From 22a3f7a56d2fdcb65d2264a4046fda39649ba0f8 Mon Sep 17 00:00:00 2001 From: Merlin Osayimwen Date: Thu, 23 Sep 2021 21:44:09 +0200 Subject: [PATCH 01/14] Start code validation --- protocol/package.gradle | 1 - runtime/build.gradle | 5 + .../InProcessExecution.java | 5 +- .../InProcessExecutionControl.java | 6 +- .../{inprocess => sandbox}/MultiTenancy.java | 2 +- .../{inprocess => sandbox}/Preemption.java | 2 +- .../environment/sandbox/SandboxLoader.java | 271 ++++++++++++++++++ .../{inprocess => sandbox}/Tenancy.java | 2 +- .../TenantBasedOutput.java | 2 +- .../jsheets/sandbox/SandboxBytecodeCheck.java | 82 ++++++ .../java/jsheets/sandbox/SandboxConfig.java | 5 + .../jsheets/sandbox/validation/Analysis.java | 7 + .../validation/ForbiddenMethodFilter.java | 10 + .../java/jsheets/sandbox/validation/Rule.java | 7 + runtime/src/main/resources/.sandbox | 4 + .../shell/ShellEvaluationEngineTest.java | 9 +- .../inprocess/TenantBasedOutputTest.java | 1 + 17 files changed, 407 insertions(+), 14 deletions(-) rename runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/{inprocess => sandbox}/InProcessExecution.java (92%) rename runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/{inprocess => sandbox}/InProcessExecutionControl.java (97%) rename runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/{inprocess => sandbox}/MultiTenancy.java (96%) rename runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/{inprocess => sandbox}/Preemption.java (73%) create mode 100644 runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/sandbox/SandboxLoader.java rename runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/{inprocess => sandbox}/Tenancy.java (67%) rename runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/{inprocess => sandbox}/TenantBasedOutput.java (98%) create mode 100644 runtime/src/main/java/jsheets/sandbox/SandboxBytecodeCheck.java create mode 100644 runtime/src/main/java/jsheets/sandbox/SandboxConfig.java create mode 100644 runtime/src/main/java/jsheets/sandbox/validation/Analysis.java create mode 100644 runtime/src/main/java/jsheets/sandbox/validation/ForbiddenMethodFilter.java create mode 100644 runtime/src/main/java/jsheets/sandbox/validation/Rule.java create mode 100644 runtime/src/main/resources/.sandbox diff --git a/protocol/package.gradle b/protocol/package.gradle index edf62b9..c2e1c18 100644 --- a/protocol/package.gradle +++ b/protocol/package.gradle @@ -217,7 +217,6 @@ final class MoreFiles { } task generateJavascriptPackage { - dependsOn generateProto doLast { def generation =new GeneratedLibrary( version: version, diff --git a/runtime/build.gradle b/runtime/build.gradle index dd95df3..5321d7a 100644 --- a/runtime/build.gradle +++ b/runtime/build.gradle @@ -12,8 +12,13 @@ repositories { mavenCentral() } +ext { + asmVersion = '9.2' +} + dependencies { implementation project(':protocol') + implementation "org.ow2.asm:asm:$asmVersion" implementation "com.google.protobuf:protobuf-java-util:$protobufJavaVersion" implementation "com.google.flogger:flogger:$floggerVersion" testImplementation "org.junit.jupiter:junit-jupiter-api:$junitPlatformVersion" diff --git a/runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/inprocess/InProcessExecution.java b/runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/sandbox/InProcessExecution.java similarity index 92% rename from runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/inprocess/InProcessExecution.java rename to runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/sandbox/InProcessExecution.java index 741f09b..214ddcb 100644 --- a/runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/inprocess/InProcessExecution.java +++ b/runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/sandbox/InProcessExecution.java @@ -1,4 +1,4 @@ -package jsheets.runtime.evaluation.shell.environment.inprocess; +package jsheets.runtime.evaluation.shell.environment.sandbox; import java.util.Map; @@ -60,7 +60,8 @@ public ExecutionControl generate( return new InProcessExecutionControl( environment, tenancy, - "schell-executor-" + name + "schell-executor-" + name, + new SandboxLoader() ); } } diff --git a/runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/inprocess/InProcessExecutionControl.java b/runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/sandbox/InProcessExecutionControl.java similarity index 97% rename from runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/inprocess/InProcessExecutionControl.java rename to runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/sandbox/InProcessExecutionControl.java index 8d63cf8..1e719a6 100644 --- a/runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/inprocess/InProcessExecutionControl.java +++ b/runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/sandbox/InProcessExecutionControl.java @@ -1,4 +1,4 @@ -package jsheets.runtime.evaluation.shell.environment.inprocess; +package jsheets.runtime.evaluation.shell.environment.sandbox; import java.lang.reflect.Method; import java.util.Objects; @@ -29,8 +29,10 @@ public final class InProcessExecutionControl extends DirectExecutionControl { InProcessExecutionControl( ExecutionEnv environment, Tenancy tenancy, - String workerGroupName + String workerGroupName, + SandboxLoader loader ) { + super(loader); this.tenancy = tenancy; this.environment = environment; this.workerGroupName = workerGroupName; diff --git a/runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/inprocess/MultiTenancy.java b/runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/sandbox/MultiTenancy.java similarity index 96% rename from runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/inprocess/MultiTenancy.java rename to runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/sandbox/MultiTenancy.java index b5c02be..773d84d 100644 --- a/runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/inprocess/MultiTenancy.java +++ b/runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/sandbox/MultiTenancy.java @@ -1,4 +1,4 @@ -package jsheets.runtime.evaluation.shell.environment.inprocess; +package jsheets.runtime.evaluation.shell.environment.sandbox; import java.io.PrintStream; import java.util.function.Consumer; diff --git a/runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/inprocess/Preemption.java b/runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/sandbox/Preemption.java similarity index 73% rename from runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/inprocess/Preemption.java rename to runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/sandbox/Preemption.java index 21daf53..5b540c4 100644 --- a/runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/inprocess/Preemption.java +++ b/runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/sandbox/Preemption.java @@ -1,4 +1,4 @@ -package jsheets.runtime.evaluation.shell.environment.inprocess; +package jsheets.runtime.evaluation.shell.environment.sandbox; /** * Thrown when the execution is preempted using {@link InProcessExecutionControl#stop()}. diff --git a/runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/sandbox/SandboxLoader.java b/runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/sandbox/SandboxLoader.java new file mode 100644 index 0000000..c95652b --- /dev/null +++ b/runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/sandbox/SandboxLoader.java @@ -0,0 +1,271 @@ +package jsheets.runtime.evaluation.shell.environment.sandbox; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.net.URLClassLoader; +import java.net.URLConnection; +import java.net.URLStreamHandler; +import java.security.CodeSource; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import jdk.jshell.execution.LoaderDelegate; +import jdk.jshell.spi.ExecutionControl; +import jsheets.sandbox.SandboxBytecodeCheck; + +public final class SandboxLoader implements LoaderDelegate { + private final SandboxLoader.RemoteClassLoader loader; + private final Map> types = new HashMap<>(); + + public SandboxLoader() { + this.loader = new RemoteClassLoader(); + Thread.currentThread().setContextClassLoader(loader); + } + + @Override + public void load(ExecutionControl.ClassBytecodes[] binaries) + throws ExecutionControl.ClassInstallException + { + loadBinaries(binaries); + preload(binaries); + } + + private void loadBinaries(ExecutionControl.ClassBytecodes[] binaries) + throws ExecutionControl.ClassInstallException + { + try { + for (var binary : binaries) { + SandboxBytecodeCheck.create().run(binary.bytecodes()); + loader.declare(binary.name(), binary.bytecodes()); + } + } catch (Throwable failure) { + throw new ExecutionControl.ClassInstallException( + "declare: " + failure.getMessage(), + new boolean[0] + ); + } + } + + private void preload(ExecutionControl.ClassBytecodes[] binaries) + throws ExecutionControl.ClassInstallException + { + boolean[] loaded = new boolean[binaries.length]; + try { + for ( int index = 0; index < binaries.length; ++index ) { + var code = binaries[index]; + var type = loader.loadClass(code.name()); + types.put(code.name(), type); + loaded[index] = true; + preload(type); + } + } catch (Throwable failure) { + throw new ExecutionControl.ClassInstallException("load: " + failure.getMessage(), + loaded + ); + } + } + + private void preload(Class type) { + type.getDeclaredMethods(); + } + + @Override + public void classesRedefined(ExecutionControl.ClassBytecodes[] binaries) { + for (var binary : binaries) { + loader.declare(binary.name(), binary.bytecodes()); + } + } + + @Override + public void addToClasspath(String classPath) throws ExecutionControl.InternalException { + try { + for (var path : classPath.split(File.pathSeparator)) { + loader.addURL(new File(path).toURI().toURL()); + } + } catch (Exception failure) { + throw new ExecutionControl.InternalException(failure.toString()); + } + } + + @Override + public Class findClass(String name) throws ClassNotFoundException { + var type = types.get(name); + if (type == null) { + throw new ClassNotFoundException(name + " not found"); + } + return type; + } + + private record ClassFile(byte[] data, long timestamp) {} + + private static class RemoteClassLoader extends URLClassLoader { + private final Map classFiles = new HashMap<>(); + + RemoteClassLoader() { + super(new URL[0]); + } + + void declare(String name, byte[] bytes) { + classFiles.put(toResourceString(name), + new ClassFile(bytes, System.currentTimeMillis()) + ); + } + + private String toResourceString(String className) { + return className.replace('.', '/') + ".class"; + } + + @Override + protected Class findClass(String name) throws ClassNotFoundException { + var file = classFiles.get(toResourceString(name)); + if (file == null) { + return super.findClass(name); + } + return super.defineClass( + name, + file.data, + 0, + file.data.length, + (CodeSource) null + ); + } + + @Override + public URL findResource(String name) { + URL u = doFindResource(name); + return u != null ? u : super.findResource(name); + } + + private URL doFindResource(String name) { + if (classFiles.containsKey(name)) { + try { + return new URL(null, + new URI("jshell", null, "/" + name, null).toString(), + new RemoteClassLoader.ResourceURLStreamHandler(name) + ); + } catch (MalformedURLException | URISyntaxException ex) { + throw new InternalError(ex); + } + } + + return null; + } + + @Override + public Enumeration findResources(String name) throws IOException { + URL u = doFindResource(name); + Enumeration sup = super.findResources(name); + + if (u == null) { + return sup; + } + + List result = new ArrayList<>(); + + while (sup.hasMoreElements()) { + result.add(sup.nextElement()); + } + + result.add(u); + + return Collections.enumeration(result); + } + + @Override + public void addURL(URL url) { + super.addURL(url); + } + + + private class ResourceURLStreamHandler extends URLStreamHandler { + private final String name; + + ResourceURLStreamHandler(String name) { + this.name = name; + } + + @Override + protected URLConnection openConnection(URL u) throws IOException { + return new URLConnection(u) { + private InputStream in; + private Map> fields; + private List fieldNames; + + @Override + public InputStream getInputStream() throws IOException { + connect(); + return in; + } + + @Override + public void connect() { + if (connected) { + return; + } + connected = true; + var file = classFiles.get(name); + in = new ByteArrayInputStream(file.data); + fields = new LinkedHashMap<>(); + fields.put( + "content-length", + List.of(Integer.toString(file.data.length)) + ); + Instant instant = new Date(file.timestamp).toInstant(); + ZonedDateTime time = ZonedDateTime.ofInstant( + instant, + ZoneId.of("GMT") + ); + String timeStamp = DateTimeFormatter.RFC_1123_DATE_TIME.format(time); + fields.put("date", List.of(timeStamp)); + fields.put("last-modified", List.of(timeStamp)); + fieldNames = new ArrayList<>(fields.keySet()); + } + + @Override + public Map> getHeaderFields() { + connect(); + return fields; + } + + @Override + public String getHeaderField(int n) { + String name = getHeaderFieldKey(n); + + return name != null ? getHeaderField(name) : null; + } + + @Override + public String getHeaderFieldKey(int n) { + return n < fieldNames.size() ? fieldNames.get(n) : null; + } + + @Override + public String getHeaderField(String name) { + connect(); + return fields.getOrDefault(name, List.of()) + .stream() + .findFirst() + .orElse(null); + } + + }; + } + } + } +} \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/inprocess/Tenancy.java b/runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/sandbox/Tenancy.java similarity index 67% rename from runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/inprocess/Tenancy.java rename to runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/sandbox/Tenancy.java index 4a3c040..a2b331d 100644 --- a/runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/inprocess/Tenancy.java +++ b/runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/sandbox/Tenancy.java @@ -1,4 +1,4 @@ -package jsheets.runtime.evaluation.shell.environment.inprocess; +package jsheets.runtime.evaluation.shell.environment.sandbox; import jdk.jshell.spi.ExecutionEnv; diff --git a/runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/inprocess/TenantBasedOutput.java b/runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/sandbox/TenantBasedOutput.java similarity index 98% rename from runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/inprocess/TenantBasedOutput.java rename to runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/sandbox/TenantBasedOutput.java index 1c20558..4b30129 100644 --- a/runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/inprocess/TenantBasedOutput.java +++ b/runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/sandbox/TenantBasedOutput.java @@ -1,4 +1,4 @@ -package jsheets.runtime.evaluation.shell.environment.inprocess; +package jsheets.runtime.evaluation.shell.environment.sandbox; import com.google.errorprone.annotations.Var; diff --git a/runtime/src/main/java/jsheets/sandbox/SandboxBytecodeCheck.java b/runtime/src/main/java/jsheets/sandbox/SandboxBytecodeCheck.java new file mode 100644 index 0000000..4d35044 --- /dev/null +++ b/runtime/src/main/java/jsheets/sandbox/SandboxBytecodeCheck.java @@ -0,0 +1,82 @@ +package jsheets.sandbox; + +import java.util.Collection; +import java.util.Objects; + +import jsheets.sandbox.validation.Analysis; +import jsheets.sandbox.validation.Rule; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; + +public final class SandboxBytecodeCheck { + public static SandboxBytecodeCheck withRules(Collection rules) { + Objects.requireNonNull(rules, "rules"); + return new SandboxBytecodeCheck(rules); + } + + private final Collection rules; + + private SandboxBytecodeCheck(Collection rules) { + this.rules = rules; + } + + public void run(byte[] classCode, Analysis analysis) { + var reader = new ClassReader(classCode); + reader.accept(new ClassCheck(rules, analysis), 0); + } + + static final class ClassCheck extends ClassVisitor { + private final Collection rules; + private final Analysis analysis; + + private ClassCheck(Collection rules, Analysis analysis) { + super(Opcodes.ASM9); + this.rules = rules; + this.analysis = analysis; + } + + @Override + public MethodVisitor visitMethod( + int access, + String name, + String descriptor, + String signature, + String[] exceptions + ) { + return new MethodCheck(name, rules, analysis); + } + } + + static final class MethodCheck extends MethodVisitor { + private final String name; + private final Collection rules; + private final Analysis analysis; + + private MethodCheck( + String name, + Collection rules, + Analysis analysis + ) { + super(Opcodes.ASM9); + this.name = name; + this.rules = rules; + this.analysis = analysis; + } + + @Override + public void visitMethodInsn( + int opcode, + String owner, + String name, + String descriptor, + boolean isInterface + ) { + var call = new Rule.MethodCall(owner, name); + for (var rule : rules) { + rule.visitCall(analysis, call); + } + } + } +} \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/sandbox/SandboxConfig.java b/runtime/src/main/java/jsheets/sandbox/SandboxConfig.java new file mode 100644 index 0000000..ed6076d --- /dev/null +++ b/runtime/src/main/java/jsheets/sandbox/SandboxConfig.java @@ -0,0 +1,5 @@ +package jsheets.sandbox; + +public final class SandboxConfig { + private SandboxConfig() {} +} \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/sandbox/validation/Analysis.java b/runtime/src/main/java/jsheets/sandbox/validation/Analysis.java new file mode 100644 index 0000000..93a92e2 --- /dev/null +++ b/runtime/src/main/java/jsheets/sandbox/validation/Analysis.java @@ -0,0 +1,7 @@ +package jsheets.sandbox.validation; + +public final class Analysis { + public record Violation() {} + + private Analysis() {} +} \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/sandbox/validation/ForbiddenMethodFilter.java b/runtime/src/main/java/jsheets/sandbox/validation/ForbiddenMethodFilter.java new file mode 100644 index 0000000..c9ab761 --- /dev/null +++ b/runtime/src/main/java/jsheets/sandbox/validation/ForbiddenMethodFilter.java @@ -0,0 +1,10 @@ +package jsheets.sandbox.validation; + +public final class ForbiddenMethodFilter implements Rule { + private ForbiddenMethodFilter() {} + + @Override + public void visitCall(Analysis analysis, MethodCall call) { + System.out.println(call); + } +} \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/sandbox/validation/Rule.java b/runtime/src/main/java/jsheets/sandbox/validation/Rule.java new file mode 100644 index 0000000..698e0b6 --- /dev/null +++ b/runtime/src/main/java/jsheets/sandbox/validation/Rule.java @@ -0,0 +1,7 @@ +package jsheets.sandbox.validation; + +public interface Rule { + record MethodCall(String owner, String method) { } + + default void visitCall(Analysis analysis, MethodCall call) {} +} diff --git a/runtime/src/main/resources/.sandbox b/runtime/src/main/resources/.sandbox new file mode 100644 index 0000000..18c1625 --- /dev/null +++ b/runtime/src/main/resources/.sandbox @@ -0,0 +1,4 @@ +java.lang.String +!java.lang.String.intern +java.lang.Objects +java.util.Arrays \ No newline at end of file diff --git a/runtime/src/test/java/jsheets/runtime/evaluation/shell/ShellEvaluationEngineTest.java b/runtime/src/test/java/jsheets/runtime/evaluation/shell/ShellEvaluationEngineTest.java index b0e2623..743e2b1 100644 --- a/runtime/src/test/java/jsheets/runtime/evaluation/shell/ShellEvaluationEngineTest.java +++ b/runtime/src/test/java/jsheets/runtime/evaluation/shell/ShellEvaluationEngineTest.java @@ -9,8 +9,7 @@ import jsheets.SnippetSources; import jsheets.StartEvaluationRequest; import jsheets.runtime.evaluation.Evaluation; -import jsheets.runtime.evaluation.shell.environment.StandardExecution; -import org.junit.jupiter.api.Disabled; +import jsheets.runtime.evaluation.shell.environment.sandbox.InProcessExecution; import org.junit.jupiter.api.Test; /* This is not a Unit Test */ @@ -23,9 +22,9 @@ static SnippetSources.CodeComponent code(String id, String content) { } @Test - @Disabled + // @Disabled public void testExecution() { - var environment = StandardExecution.create(); + var environment = InProcessExecution.create(); var installation = environment.install(); var engine = ShellEvaluationEngine.newBuilder() .useClock(Clock.systemUTC()) @@ -55,9 +54,9 @@ public void testExecution() { .addCodeComponents(code("1", "1 + 2")) .addCodeComponents(code("2", "int x = 10;")) .addCodeComponents(code("3", "x * x")) + .addCodeComponents(code("6", "new Test().toString()")) .addCodeComponents(code("4", "lol")) .addCodeComponents(code("5", "class Test { }")) - .addCodeComponents(code("6", "new Test().toString()")) .addCodeComponents(code("7", """ System.out.println("Hello, World!"); System.out.println("Hello, World!"); diff --git a/runtime/src/test/java/jsheets/runtime/evaluation/shell/environment/inprocess/TenantBasedOutputTest.java b/runtime/src/test/java/jsheets/runtime/evaluation/shell/environment/inprocess/TenantBasedOutputTest.java index 997404f..6c91c5f 100644 --- a/runtime/src/test/java/jsheets/runtime/evaluation/shell/environment/inprocess/TenantBasedOutputTest.java +++ b/runtime/src/test/java/jsheets/runtime/evaluation/shell/environment/inprocess/TenantBasedOutputTest.java @@ -2,6 +2,7 @@ import com.google.common.util.concurrent.Uninterruptibles; +import jsheets.runtime.evaluation.shell.environment.sandbox.TenantBasedOutput; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; From eb4ef157bdd7553d8f075b4ab2af0e29bbfbcfbd Mon Sep 17 00:00:00 2001 From: Merlin Osayimwen Date: Sun, 10 Oct 2021 02:37:03 +0200 Subject: [PATCH 02/14] Change config --- runtime/build.gradle | 3 + .../evaluation/shell/ShellEvaluation.java | 1 - .../java/jsheets/sandbox/AccessGraph.java | 292 ++++++++++++++++++ .../java/jsheets/sandbox/MethodSignature.java | 84 +++++ .../java/jsheets/sandbox/SandboxConfig.java | 2 + .../sandbox/SandboxLoader.java | 144 ++++----- .../jsheets/sandbox/SandboxedEnvironment.java | 29 ++ runtime/src/main/resources/.sandbox | 5 +- .../java/jsheets/sandbox/AccessGraphTest.java | 66 ++++ .../jsheets/sandbox/MethodSignatureTest.java | 30 ++ .../jsheets/sandbox/PermissionGraphTest.java | 18 ++ .../jsheets/sandbox/SandboxExecutionTest.java | 22 ++ runtime/src/test/resources/access-graph.txt | 3 + 13 files changed, 626 insertions(+), 73 deletions(-) create mode 100644 runtime/src/main/java/jsheets/sandbox/AccessGraph.java create mode 100644 runtime/src/main/java/jsheets/sandbox/MethodSignature.java rename runtime/src/main/java/jsheets/{runtime/evaluation/shell/environment => }/sandbox/SandboxLoader.java (63%) create mode 100644 runtime/src/main/java/jsheets/sandbox/SandboxedEnvironment.java create mode 100644 runtime/src/test/java/jsheets/sandbox/AccessGraphTest.java create mode 100644 runtime/src/test/java/jsheets/sandbox/MethodSignatureTest.java create mode 100644 runtime/src/test/java/jsheets/sandbox/PermissionGraphTest.java create mode 100644 runtime/src/test/java/jsheets/sandbox/SandboxExecutionTest.java create mode 100644 runtime/src/test/resources/access-graph.txt diff --git a/runtime/build.gradle b/runtime/build.gradle index 73685dd..0f5be1e 100644 --- a/runtime/build.gradle +++ b/runtime/build.gradle @@ -14,6 +14,7 @@ repositories { ext { asmVersion = '9.2' + recordBuilderVersion = '26' } dependencies { @@ -24,6 +25,8 @@ dependencies { testImplementation "org.junit.jupiter:junit-jupiter-api:$junitPlatformVersion" testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:$junitPlatformVersion" testRuntimeOnly "com.google.flogger:flogger-slf4j-backend:$floggerVersion" + compileOnly "io.soabase.record-builder:record-builder-core:$recordBuilderVersion" + annotationProcessor "io.soabase.record-builder:record-builder-processor:$recordBuilderVersion" } test { diff --git a/runtime/src/main/java/jsheets/runtime/evaluation/shell/ShellEvaluation.java b/runtime/src/main/java/jsheets/runtime/evaluation/shell/ShellEvaluation.java index e8ad25c..084fc8f 100644 --- a/runtime/src/main/java/jsheets/runtime/evaluation/shell/ShellEvaluation.java +++ b/runtime/src/main/java/jsheets/runtime/evaluation/shell/ShellEvaluation.java @@ -1,6 +1,5 @@ package jsheets.runtime.evaluation.shell; - import java.time.Clock; import java.util.Comparator; import java.util.Locale; diff --git a/runtime/src/main/java/jsheets/sandbox/AccessGraph.java b/runtime/src/main/java/jsheets/sandbox/AccessGraph.java new file mode 100644 index 0000000..485d683 --- /dev/null +++ b/runtime/src/main/java/jsheets/sandbox/AccessGraph.java @@ -0,0 +1,292 @@ +package jsheets.sandbox; + +import com.google.errorprone.annotations.CanIgnoreReturnValue; + +import java.util.*; +import java.util.regex.Pattern; + +public final class AccessGraph { + private static final AccessGraph empty = + new AccessGraph(new Node.Path("root", Access.Denied, List.of())); + + public static AccessGraph empty() { + return empty; + } + + public static AccessGraph of(String... permissions) { + var builder = newBuilder(); + for (var permission : permissions) { + if (permission.startsWith("!")) { + var remaining = permission.substring(1).trim(); + builder.deny(Key.infer(remaining)); + } else { + builder.permit(Key.infer(permission.trim())); + } + } + return builder.create(); + } + + public enum Access { + Permitted, Denied + } + + /* sealed */ interface Node extends Iterable { + Access access(); + String name(); + Collection children(); + void insertChild(Node node); + + default Iterator iterator() { + return children().iterator(); + } + + record Path(String name, Access access, Collection children) implements Node { + public Collection listMethodsByName(String name) { + var methods = new ArrayList(); + for (var child : children) { + if (child instanceof Method method && method.name().equals(name)) { + methods.add(method); + } + } + return methods; + } + + @Override + public void insertChild(Node node) { + children.add(node); + } + } + + record Method(MethodSignature signature, Access access) implements Node { + @Override + public String name() { + return signature.methodName(); + } + + @Override + public Collection children() { + return List.of(); + } + + @Override + public void insertChild(Node node) { + throw new UnsupportedOperationException("can not add child to method"); + } + } + } + + public record Key(Pattern separator, String value) { + public static Key infer(String value) { + return value.contains("/") ? slashSeparated(value) : dotSeparated(value); + } + + private static final Pattern dotOrMethodSeparator = + Pattern.compile("[.#]"); + + public static Key dotSeparated(String value) { + Objects.requireNonNull(value, "value"); + return new Key(dotOrMethodSeparator, value); + } + + private static final Pattern slashOrMethodSeparator = + Pattern.compile("[/#]"); + + public static Key slashSeparated(String value) { + Objects.requireNonNull(value, "value"); + return new Key(slashOrMethodSeparator, value); + } + + public String[] split() { + return separator.split(value); + } + + public String lastPart() { + var parts = split(); + if (parts.length == 0) { + return value; + } + return parts[parts.length - 1]; + } + } + + private final Node root; + + private AccessGraph(Node root) { + this.root = root; + } + + private Collection findClosestMatch(Key key) { + return findClosestMatch(root, key.split(), 0); + } + + private static Collection findClosestMatch(Node node, String[] key, int depth) { + if (depth >= key.length) { + return List.of(node); + } + if (depth == key.length - 1) { + return findClosestMatchOrMethods(node, key[key.length - 1]); + } + return findClosestChild(node, key, depth); + } + + private static Collection findClosestChild(Node node, String[] key, int depth) { + for (var child : node.children()) { + if (child.name().equals(key[depth])) { + return findClosestMatch(child, key, depth + 1); + } + } + return List.of(node); + } + + private static Collection findClosestMatchOrMethods(Node node, String lastKey) { + var children = node.children().stream() + .filter(child -> child.name().equals(lastKey)) + .toList(); + return children.isEmpty() ? List.of(node) : children; + } + + /** + * Checks if the use of the class or method under the given key is permitted. + *

+ * When checking access to methods prefer {@link AccessGraph#isMethodPermitted(MethodSignature)}. + * + * @param key Key of the checked package, class or method. + * @return True if the caller has permission to use classes or methods + * under this key. + */ + public boolean isPermitted(Key key) { + var matches = findClosestMatch(key); + return !matches.isEmpty() + && matches.iterator().next().access().equals(Access.Permitted); + } + + /** + * Checks if a call to the method is permitted. + *

+ * This call is better for checking access to methods because it supports + * overloaded methods. The {@link AccessGraph#isPermitted} method does + * not handle ambiguous calls well. + *

+ * Note, that this method does not support covariant return types. + * + * @param signature Signature of the method that is checked. + * @return True if the caller has permissions to call this method. + */ + public boolean isMethodPermitted(MethodSignature signature) { + var key = Key.slashSeparated(signature.formatWithoutTypes()); + var matches = findClosestMatch(key); + var access = switch (matches.size()) { + case 0 -> Access.Denied; + case 1 -> matches.iterator().next().access(); + default -> findBestMatch(signature, matches).map(Node::access); + }; + return access.equals(Access.Permitted); + } + + private Optional findBestMatch(MethodSignature signature, Collection nodes) { + for (var node : nodes) { + if (node instanceof Node.Method method + && method.signature.matches(signature)) { + return Optional.of(node); + } + } + return Optional.empty(); + } + + @Override + public String toString() { + return "AccessGraph(%s)".formatted(root.children().toString()); + } + + @Override + public boolean equals(Object target) { + if (this == target) { + return true; + } + return target instanceof AccessGraph graph && ( + Objects.equals(graph.root, root) + ); + } + + @Override + public int hashCode() { + return Objects.hash(root); + } + + public static Builder newBuilder() { + return new Builder(); + } + + public static final class Builder { + private final Node root = + new Node.Path("root", Access.Denied, new ArrayList<>()); + + private Builder() {} + + @CanIgnoreReturnValue + public Builder permit(Key key) { + var node = createNode(key, Access.Permitted); + insert(key, node); + return this; + } + + @CanIgnoreReturnValue + public Builder deny(Key key) { + var node = createNode(key, Access.Denied); + insert(key, node); + return this; + } + + private void insert(Key key, Node node) { + var parent = resolveParent(key, node); + parent.insertChild(node); + } + + private Node resolveParent(Key key, Node inserted) { + var path = key.split(); + if (path.length == 1) { + return root; + } + var parentPath = Arrays.copyOf(path, path.length - 1); + return resolveParent(root, parentPath, 0, inserted); + } + + private Node resolveParent(Node target, String[] key, int depth, Node inserted) { + if (depth >= key.length) { + return target; + } + for (var child : target.children()) { + if (child.name().equals(key[depth])) { + return resolveParent(child, key, depth + 1, inserted); + } + } + return insertRemainingPath(target, key, depth, inserted.access()); + } + + private Node insertRemainingPath(Node target, String[] key, int depth, Access access) { + for (int index = depth; index < key.length; index++) { + var intermediate = new Node.Path(key[index], access, new ArrayList<>()); + target.insertChild(intermediate); + target = intermediate; + } + return target; + } + + private Node createNode(Key key, Access access) { + if (isMethodSignature(key.value())) { + var signature = MethodSignature.parse(key.value()); + return new Node.Method(signature, access); + } + return new Node.Path(key.lastPart(), access, new ArrayList<>()); + } + + private static boolean isMethodSignature(String path) { + return path.indexOf(MethodSignature.methodSeparator()) >= 0; + } + + // TODO: Create defensive copies after building + public AccessGraph create() { + return new AccessGraph(root); + } + } +} diff --git a/runtime/src/main/java/jsheets/sandbox/MethodSignature.java b/runtime/src/main/java/jsheets/sandbox/MethodSignature.java new file mode 100644 index 0000000..ba1fc6d --- /dev/null +++ b/runtime/src/main/java/jsheets/sandbox/MethodSignature.java @@ -0,0 +1,84 @@ +package jsheets.sandbox; + +import io.soabase.recordbuilder.core.RecordBuilder; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.regex.Pattern; + +@RecordBuilder +record MethodSignature( + String className, + String methodName, + String returnType, + Collection parameterTypes +) { + private static final char methodSeparator = '#'; + + public static char methodSeparator() { + return methodSeparator; + } + + private static final String defaultReturnType = "void"; + + public static MethodSignature parse(String input) { + var trimmed = input.trim(); + int returnTypeEnd = findBeforeMethodName(trimmed, ' '); + if (returnTypeEnd < 0) { + return parseWithReturnType(input, defaultReturnType); + } + var returnType = trimmed.substring(0, returnTypeEnd); + var remaining = trimmed.substring(returnTypeEnd + 1); + return parseWithReturnType(remaining, returnType); + } + + private static int findBeforeMethodName(String input, char character) { + int firstSpace = input.indexOf(character); + int methodNameBegin = input.indexOf(methodSeparator); + return firstSpace < methodNameBegin ? firstSpace : -1; + } + + private static MethodSignature parseWithReturnType(String input, String returnType) { + var methodNameBegin = input.indexOf(methodSeparator); + if (methodNameBegin < 0) { + throw new IllegalArgumentException( + "input does not contain method name (separated by #)" + ); + } + var className = input.substring(0, methodNameBegin); + var parameterListBegin = input.indexOf('('); + if (parameterListBegin < 0) { + var methodName = input.substring(methodNameBegin + 1); + return new MethodSignature(className, methodName, returnType, List.of()); + } + var parameterListEnd = input.lastIndexOf(')'); + var methodName = input.substring(methodNameBegin + 1, parameterListBegin); + var parameterPart = input.substring(parameterListBegin + 1, parameterListEnd); + var parameterTypes = parseParameterTypes(parameterPart); + return new MethodSignature(className, methodName, returnType, parameterTypes); + } + + private static final Pattern parameterTypeSeparator = Pattern.compile(",\\s*"); + + private static Collection parseParameterTypes(String input) { + return List.of(parameterTypeSeparator.split(input)); + } + + public boolean matches(MethodSignature signature) { + return equals(signature); + } + + public String format() { + return "%s %s#%s(%s)".formatted( + returnType, + className, + methodName, + Arrays.toString(parameterTypes.toArray()) + ); + } + + public String formatWithoutTypes() { + return "%s#%s".formatted(className, methodName); + } +} diff --git a/runtime/src/main/java/jsheets/sandbox/SandboxConfig.java b/runtime/src/main/java/jsheets/sandbox/SandboxConfig.java index ed6076d..3dadad9 100644 --- a/runtime/src/main/java/jsheets/sandbox/SandboxConfig.java +++ b/runtime/src/main/java/jsheets/sandbox/SandboxConfig.java @@ -2,4 +2,6 @@ public final class SandboxConfig { private SandboxConfig() {} + + } \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/sandbox/SandboxLoader.java b/runtime/src/main/java/jsheets/sandbox/SandboxLoader.java similarity index 63% rename from runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/sandbox/SandboxLoader.java rename to runtime/src/main/java/jsheets/sandbox/SandboxLoader.java index a76f552..f309f4d 100644 --- a/runtime/src/main/java/jsheets/runtime/evaluation/shell/environment/sandbox/SandboxLoader.java +++ b/runtime/src/main/java/jsheets/sandbox/SandboxLoader.java @@ -1,4 +1,4 @@ -package jsheets.runtime.evaluation.shell.environment.sandbox; +package jsheets.sandbox; import java.io.ByteArrayInputStream; import java.io.File; @@ -12,9 +12,7 @@ import java.net.URLConnection; import java.net.URLStreamHandler; import java.security.CodeSource; -import java.time.Instant; import java.time.ZoneId; -import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Collections; @@ -27,16 +25,22 @@ import jdk.jshell.execution.LoaderDelegate; import jdk.jshell.spi.ExecutionControl; -import jsheets.sandbox.SandboxBytecodeCheck; import jsheets.sandbox.validation.Analysis; import jsheets.sandbox.validation.ForbiddenMethodFilter; public final class SandboxLoader implements LoaderDelegate { + public static SandboxLoader create() { + return new SandboxLoader(); + } + private final SandboxLoader.RemoteClassLoader loader; private final Map> types = new HashMap<>(); - public SandboxLoader() { + private SandboxLoader() { this.loader = new RemoteClassLoader(); + } + + public void install() { Thread.currentThread().setContextClassLoader(loader); } @@ -116,78 +120,77 @@ public Class findClass(String name) throws ClassNotFoundException { return type; } - private record ClassFile(byte[] data, long timestamp) {} + private record FileRecord(byte[] content, long timestamp) {} private static class RemoteClassLoader extends URLClassLoader { - private final Map classFiles = new HashMap<>(); + private final Map files = new HashMap<>(); RemoteClassLoader() { super(new URL[0]); } void declare(String name, byte[] bytes) { - classFiles.put(toResourceString(name), - new ClassFile(bytes, System.currentTimeMillis()) + files.put( + createResourceKeyForClassName(name), + new FileRecord(bytes, System.currentTimeMillis()) ); } - private String toResourceString(String className) { - return className.replace('.', '/') + ".class"; + private String createResourceKeyForClassName(String name) { + return name.replace('.', '/') + ".class"; } @Override protected Class findClass(String name) throws ClassNotFoundException { - var file = classFiles.get(toResourceString(name)); + var file = files.get(createResourceKeyForClassName(name)); if (file == null) { return super.findClass(name); } return super.defineClass( name, - file.data, + file.content, 0, - file.data.length, + file.content.length, (CodeSource) null ); } @Override public URL findResource(String name) { - URL u = doFindResource(name); - return u != null ? u : super.findResource(name); + var resource = lookupResource(name); + return resource != null ? resource : super.findResource(name); } - private URL doFindResource(String name) { - if (classFiles.containsKey(name)) { - try { - return new URL(null, - new URI("jshell", null, "/" + name, null).toString(), - new RemoteClassLoader.ResourceURLStreamHandler(name) - ); - } catch (MalformedURLException | URISyntaxException ex) { - throw new InternalError(ex); - } + private URL lookupResource(String name) { + if (!files.containsKey(name)) { + return null; + } + try { + return new URL( + /* context */ null, + new URI("jshell", null, "/" + name, null).toString(), + new ResourceUrlStreamHandler(name) + ); + } catch (MalformedURLException | URISyntaxException failure) { + throw new InternalError(failure); } - - return null; } @Override public Enumeration findResources(String name) throws IOException { - URL u = doFindResource(name); - Enumeration sup = super.findResources(name); - - if (u == null) { - return sup; - } - - List result = new ArrayList<>(); + var resource = lookupResource(name); + var parentResources = super.findResources(name); + return resource == null + ? parentResources + : plus(parentResources, resource); + } - while (sup.hasMoreElements()) { - result.add(sup.nextElement()); + private static Enumeration plus(Enumeration enumeration, T element) { + var result = new ArrayList(); + while (enumeration.hasMoreElements()) { + result.add(enumeration.nextElement()); } - - result.add(u); - + result.add(element); return Collections.enumeration(result); } @@ -196,25 +199,24 @@ public void addURL(URL url) { super.addURL(url); } - - private class ResourceURLStreamHandler extends URLStreamHandler { + private class ResourceUrlStreamHandler extends URLStreamHandler { private final String name; - ResourceURLStreamHandler(String name) { + ResourceUrlStreamHandler(String name) { this.name = name; } @Override - protected URLConnection openConnection(URL u) throws IOException { - return new URLConnection(u) { - private InputStream in; + protected URLConnection openConnection(URL resource) { + return new URLConnection(resource) { + private InputStream input; private Map> fields; private List fieldNames; @Override - public InputStream getInputStream() throws IOException { + public InputStream getInputStream() { connect(); - return in; + return input; } @Override @@ -223,24 +225,30 @@ public void connect() { return; } connected = true; - var file = classFiles.get(name); - in = new ByteArrayInputStream(file.data); + var file = files.get(name); + input = new ByteArrayInputStream(file.content); fields = new LinkedHashMap<>(); - fields.put( - "content-length", - List.of(Integer.toString(file.data.length)) - ); - Instant instant = new Date(file.timestamp).toInstant(); - ZonedDateTime time = ZonedDateTime.ofInstant( - instant, - ZoneId.of("GMT") - ); - String timeStamp = DateTimeFormatter.RFC_1123_DATE_TIME.format(time); - fields.put("date", List.of(timeStamp)); - fields.put("last-modified", List.of(timeStamp)); + initializeHeaders(fields, file); fieldNames = new ArrayList<>(fields.keySet()); } + private void initializeHeaders( + Map> headers, + FileRecord file + ) { + var length = Integer.toString(file.content.length); + headers.put("content-length", List.of(length)); + var timeStamp = formatTime(file.timestamp); + headers.put("date", List.of(timeStamp)); + headers.put("last-modified", List.of(timeStamp)); + } + + private String formatTime(long timeStamp) { + var instant = new Date(timeStamp).toInstant(); + var time = instant.atZone(ZoneId.of("GMT")); + return DateTimeFormatter.RFC_1123_DATE_TIME.format(time); + } + @Override public Map> getHeaderFields() { connect(); @@ -248,15 +256,14 @@ public Map> getHeaderFields() { } @Override - public String getHeaderField(int n) { - String name = getHeaderFieldKey(n); - + public String getHeaderField(int index) { + var name = getHeaderFieldKey(index); return name != null ? getHeaderField(name) : null; } @Override - public String getHeaderFieldKey(int n) { - return n < fieldNames.size() ? fieldNames.get(n) : null; + public String getHeaderFieldKey(int index) { + return index < fieldNames.size() ? fieldNames.get(index) : null; } @Override @@ -267,7 +274,6 @@ public String getHeaderField(String name) { .findFirst() .orElse(null); } - }; } } diff --git a/runtime/src/main/java/jsheets/sandbox/SandboxedEnvironment.java b/runtime/src/main/java/jsheets/sandbox/SandboxedEnvironment.java new file mode 100644 index 0000000..c5b1667 --- /dev/null +++ b/runtime/src/main/java/jsheets/sandbox/SandboxedEnvironment.java @@ -0,0 +1,29 @@ +package jsheets.sandbox; + +import jdk.jshell.execution.DirectExecutionControl; +import jdk.jshell.spi.ExecutionControl; +import jdk.jshell.spi.ExecutionControlProvider; +import jdk.jshell.spi.ExecutionEnv; + +import java.util.Map; + +public final class SandboxedEnvironment implements ExecutionControlProvider { + public static SandboxedEnvironment create() { + return new SandboxedEnvironment(); + } + + private SandboxedEnvironment() {} + + @Override + public String name() { + return "sandbox"; + } + + @Override + public ExecutionControl generate( + ExecutionEnv environment, + Map parameters + ) { + return new DirectExecutionControl(SandboxLoader.create()); + } +} diff --git a/runtime/src/main/resources/.sandbox b/runtime/src/main/resources/.sandbox index 18c1625..991b958 100644 --- a/runtime/src/main/resources/.sandbox +++ b/runtime/src/main/resources/.sandbox @@ -1,4 +1,3 @@ +java.lang.* java.lang.String -!java.lang.String.intern -java.lang.Objects -java.util.Arrays \ No newline at end of file +! void java.lang.String#intern() diff --git a/runtime/src/test/java/jsheets/sandbox/AccessGraphTest.java b/runtime/src/test/java/jsheets/sandbox/AccessGraphTest.java new file mode 100644 index 0000000..89d6960 --- /dev/null +++ b/runtime/src/test/java/jsheets/sandbox/AccessGraphTest.java @@ -0,0 +1,66 @@ +package jsheets.sandbox; + +import com.google.common.base.Charsets; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.List; + +import static jsheets.sandbox.AccessGraph.Key.*; + +public class AccessGraphTest { + + @Test + public void testReadingFromFile() { + var specification = readFileContent("access-graph.txt"); + var graph = AccessGraph.of(specification.split("\n")); + Assertions.assertTrue(graph.isPermitted(dotSeparated("a.b.c"))); + Assertions.assertTrue(graph.isPermitted(dotSeparated("a.b.c.Foo"))); + Assertions.assertTrue(graph.isPermitted(dotSeparated("a.b.c.Foo#run"))); + Assertions.assertTrue(graph.isPermitted(dotSeparated("a.b.c.Unknown"))); + Assertions.assertFalse(graph.isPermitted(dotSeparated("a.b.c.Foo#exit"))); + Assertions.assertFalse(graph.isPermitted(dotSeparated("a.b.c.Bar"))); + Assertions.assertFalse(graph.isPermitted(dotSeparated("a.b.c.Bar#run"))); + } + + @Test + public void testOverloadedMethod() { + var graph = AccessGraph.of( + "void a.b.c.Foo#run(String[], int)" + ); + Assertions.assertTrue(graph.isMethodPermitted(MethodSignature.parse("void a.b.c.Foo#run(String[], int)"))); + Assertions.assertFalse(graph.isMethodPermitted(MethodSignature.parse("void a.b.c.Foo#run"))); + } + + @Test + public void testKeySplit() { + Assertions.assertEquals( + List.of("a", "b", "c"), + List.of(dotSeparated("a.b.c").split()) + ); + Assertions.assertEquals( + List.of("a", "b", "c"), + List.of(slashSeparated("a/b/c").split()) + ); + Assertions.assertEquals( + List.of("a", "b", "c", "Foo"), + List.of(slashSeparated("a/b/c#Foo").split()) + ); + } + + private String readFileContent(String path) { + var resource = Thread.currentThread() + .getContextClassLoader() + .getResourceAsStream(path); + if (resource == null) { + throw new RuntimeException(new FileNotFoundException()); + } + try (resource) { + return new String(resource.readAllBytes(), Charsets.UTF_8); + } catch (IOException failure) { + throw new RuntimeException(failure); + } + } +} diff --git a/runtime/src/test/java/jsheets/sandbox/MethodSignatureTest.java b/runtime/src/test/java/jsheets/sandbox/MethodSignatureTest.java new file mode 100644 index 0000000..a5a6898 --- /dev/null +++ b/runtime/src/test/java/jsheets/sandbox/MethodSignatureTest.java @@ -0,0 +1,30 @@ +package jsheets.sandbox; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; + +public class MethodSignatureTest { + @Test + public void testParsing() { + Assertions.assertEquals( + MethodSignatureBuilder.builder() + .className("java/lang/System") + .methodName("exit") + .returnType("void") + .parameterTypes(List.of("int")) + .build(), + MethodSignature.parse("void java/lang/System#exit(int)") + ); + Assertions.assertEquals( + MethodSignatureBuilder.builder() + .className("java/lang/String") + .methodName("join") + .returnType("void") + .parameterTypes(List.of("java/lang/CharSequence", "java/lang/Iterable")) + .build(), + MethodSignature.parse("java/lang/String#join(java/lang/CharSequence, java/lang/Iterable)") + ); + } +} diff --git a/runtime/src/test/java/jsheets/sandbox/PermissionGraphTest.java b/runtime/src/test/java/jsheets/sandbox/PermissionGraphTest.java new file mode 100644 index 0000000..e53758b --- /dev/null +++ b/runtime/src/test/java/jsheets/sandbox/PermissionGraphTest.java @@ -0,0 +1,18 @@ +package jsheets.sandbox; + +import org.junit.jupiter.api.Assertions; + +import static jsheets.sandbox.AccessGraph.Key.slashSeparated; + +public class PermissionGraphTest { + public void testDirectPath() { + var graph = AccessGraph.of( + "java.lang", + "java.util.List", + "void java.lang.Thread#sleep()" + ); + Assertions.assertTrue(graph.isPermitted(slashSeparated("java/lang"))); + Assertions.assertTrue(graph.isPermitted(slashSeparated("java/lang"))); + Assertions.assertTrue(graph.isPermitted(slashSeparated("java/lang"))); + } +} diff --git a/runtime/src/test/java/jsheets/sandbox/SandboxExecutionTest.java b/runtime/src/test/java/jsheets/sandbox/SandboxExecutionTest.java new file mode 100644 index 0000000..c2d77e5 --- /dev/null +++ b/runtime/src/test/java/jsheets/sandbox/SandboxExecutionTest.java @@ -0,0 +1,22 @@ +package jsheets.sandbox; + +import jdk.jshell.JShell; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +public final class SandboxExecutionTest { + @Test + public void test() { + var shell = createSandboxedShell(); + shell.eval("System.out.println(\"Hello, World!\");"); + } + + private JShell createSandboxedShell() { + return JShell.builder() + .out(System.out) + .err(System.err) + .executionEngine(SandboxedEnvironment.create(), Map.of()) + .build(); + } +} diff --git a/runtime/src/test/resources/access-graph.txt b/runtime/src/test/resources/access-graph.txt new file mode 100644 index 0000000..71243fa --- /dev/null +++ b/runtime/src/test/resources/access-graph.txt @@ -0,0 +1,3 @@ +a.b.c.Foo +!a.b.c.Foo#exit +!a.b.c.Bar \ No newline at end of file From f6ec4eda0d5dcd7a071919f0e477b5835f79b232 Mon Sep 17 00:00:00 2001 From: Merlin Osayimwen Date: Wed, 13 Oct 2021 22:33:43 +0200 Subject: [PATCH 03/14] Change AccessGraph --- .../java/jsheets/sandbox/AccessGraph.java | 292 ------------------ .../jsheets/sandbox/SandboxBytecodeCheck.java | 34 +- .../java/jsheets/sandbox/SandboxLoader.java | 28 +- .../jsheets/sandbox/SandboxedEnvironment.java | 16 +- .../java/jsheets/sandbox/access/Access.java | 7 + .../jsheets/sandbox/access/AccessGraph.java | 154 +++++++++ .../sandbox/access/AccessGraphBuilder.java | 90 ++++++ .../sandbox/access/AccessGraphNode.java | 137 ++++++++ .../jsheets/sandbox/access/AccessKey.java | 38 +++ .../sandbox/{ => access}/MethodSignature.java | 14 +- .../jsheets/sandbox/validation/Analysis.java | 27 +- .../validation/ForbiddenMemberFilter.java | 48 +++ .../validation/ForbiddenMethodFilter.java | 14 - .../java/jsheets/sandbox/validation/Rule.java | 5 +- .../java/jsheets/sandbox/AccessGraphTest.java | 6 +- .../jsheets/sandbox/MethodSignatureTest.java | 2 + .../jsheets/sandbox/PermissionGraphTest.java | 10 +- .../jsheets/sandbox/SandboxExecutionTest.java | 20 +- 18 files changed, 607 insertions(+), 335 deletions(-) delete mode 100644 runtime/src/main/java/jsheets/sandbox/AccessGraph.java create mode 100644 runtime/src/main/java/jsheets/sandbox/access/Access.java create mode 100644 runtime/src/main/java/jsheets/sandbox/access/AccessGraph.java create mode 100644 runtime/src/main/java/jsheets/sandbox/access/AccessGraphBuilder.java create mode 100644 runtime/src/main/java/jsheets/sandbox/access/AccessGraphNode.java create mode 100644 runtime/src/main/java/jsheets/sandbox/access/AccessKey.java rename runtime/src/main/java/jsheets/sandbox/{ => access}/MethodSignature.java (90%) create mode 100644 runtime/src/main/java/jsheets/sandbox/validation/ForbiddenMemberFilter.java delete mode 100644 runtime/src/main/java/jsheets/sandbox/validation/ForbiddenMethodFilter.java diff --git a/runtime/src/main/java/jsheets/sandbox/AccessGraph.java b/runtime/src/main/java/jsheets/sandbox/AccessGraph.java deleted file mode 100644 index 485d683..0000000 --- a/runtime/src/main/java/jsheets/sandbox/AccessGraph.java +++ /dev/null @@ -1,292 +0,0 @@ -package jsheets.sandbox; - -import com.google.errorprone.annotations.CanIgnoreReturnValue; - -import java.util.*; -import java.util.regex.Pattern; - -public final class AccessGraph { - private static final AccessGraph empty = - new AccessGraph(new Node.Path("root", Access.Denied, List.of())); - - public static AccessGraph empty() { - return empty; - } - - public static AccessGraph of(String... permissions) { - var builder = newBuilder(); - for (var permission : permissions) { - if (permission.startsWith("!")) { - var remaining = permission.substring(1).trim(); - builder.deny(Key.infer(remaining)); - } else { - builder.permit(Key.infer(permission.trim())); - } - } - return builder.create(); - } - - public enum Access { - Permitted, Denied - } - - /* sealed */ interface Node extends Iterable { - Access access(); - String name(); - Collection children(); - void insertChild(Node node); - - default Iterator iterator() { - return children().iterator(); - } - - record Path(String name, Access access, Collection children) implements Node { - public Collection listMethodsByName(String name) { - var methods = new ArrayList(); - for (var child : children) { - if (child instanceof Method method && method.name().equals(name)) { - methods.add(method); - } - } - return methods; - } - - @Override - public void insertChild(Node node) { - children.add(node); - } - } - - record Method(MethodSignature signature, Access access) implements Node { - @Override - public String name() { - return signature.methodName(); - } - - @Override - public Collection children() { - return List.of(); - } - - @Override - public void insertChild(Node node) { - throw new UnsupportedOperationException("can not add child to method"); - } - } - } - - public record Key(Pattern separator, String value) { - public static Key infer(String value) { - return value.contains("/") ? slashSeparated(value) : dotSeparated(value); - } - - private static final Pattern dotOrMethodSeparator = - Pattern.compile("[.#]"); - - public static Key dotSeparated(String value) { - Objects.requireNonNull(value, "value"); - return new Key(dotOrMethodSeparator, value); - } - - private static final Pattern slashOrMethodSeparator = - Pattern.compile("[/#]"); - - public static Key slashSeparated(String value) { - Objects.requireNonNull(value, "value"); - return new Key(slashOrMethodSeparator, value); - } - - public String[] split() { - return separator.split(value); - } - - public String lastPart() { - var parts = split(); - if (parts.length == 0) { - return value; - } - return parts[parts.length - 1]; - } - } - - private final Node root; - - private AccessGraph(Node root) { - this.root = root; - } - - private Collection findClosestMatch(Key key) { - return findClosestMatch(root, key.split(), 0); - } - - private static Collection findClosestMatch(Node node, String[] key, int depth) { - if (depth >= key.length) { - return List.of(node); - } - if (depth == key.length - 1) { - return findClosestMatchOrMethods(node, key[key.length - 1]); - } - return findClosestChild(node, key, depth); - } - - private static Collection findClosestChild(Node node, String[] key, int depth) { - for (var child : node.children()) { - if (child.name().equals(key[depth])) { - return findClosestMatch(child, key, depth + 1); - } - } - return List.of(node); - } - - private static Collection findClosestMatchOrMethods(Node node, String lastKey) { - var children = node.children().stream() - .filter(child -> child.name().equals(lastKey)) - .toList(); - return children.isEmpty() ? List.of(node) : children; - } - - /** - * Checks if the use of the class or method under the given key is permitted. - *

- * When checking access to methods prefer {@link AccessGraph#isMethodPermitted(MethodSignature)}. - * - * @param key Key of the checked package, class or method. - * @return True if the caller has permission to use classes or methods - * under this key. - */ - public boolean isPermitted(Key key) { - var matches = findClosestMatch(key); - return !matches.isEmpty() - && matches.iterator().next().access().equals(Access.Permitted); - } - - /** - * Checks if a call to the method is permitted. - *

- * This call is better for checking access to methods because it supports - * overloaded methods. The {@link AccessGraph#isPermitted} method does - * not handle ambiguous calls well. - *

- * Note, that this method does not support covariant return types. - * - * @param signature Signature of the method that is checked. - * @return True if the caller has permissions to call this method. - */ - public boolean isMethodPermitted(MethodSignature signature) { - var key = Key.slashSeparated(signature.formatWithoutTypes()); - var matches = findClosestMatch(key); - var access = switch (matches.size()) { - case 0 -> Access.Denied; - case 1 -> matches.iterator().next().access(); - default -> findBestMatch(signature, matches).map(Node::access); - }; - return access.equals(Access.Permitted); - } - - private Optional findBestMatch(MethodSignature signature, Collection nodes) { - for (var node : nodes) { - if (node instanceof Node.Method method - && method.signature.matches(signature)) { - return Optional.of(node); - } - } - return Optional.empty(); - } - - @Override - public String toString() { - return "AccessGraph(%s)".formatted(root.children().toString()); - } - - @Override - public boolean equals(Object target) { - if (this == target) { - return true; - } - return target instanceof AccessGraph graph && ( - Objects.equals(graph.root, root) - ); - } - - @Override - public int hashCode() { - return Objects.hash(root); - } - - public static Builder newBuilder() { - return new Builder(); - } - - public static final class Builder { - private final Node root = - new Node.Path("root", Access.Denied, new ArrayList<>()); - - private Builder() {} - - @CanIgnoreReturnValue - public Builder permit(Key key) { - var node = createNode(key, Access.Permitted); - insert(key, node); - return this; - } - - @CanIgnoreReturnValue - public Builder deny(Key key) { - var node = createNode(key, Access.Denied); - insert(key, node); - return this; - } - - private void insert(Key key, Node node) { - var parent = resolveParent(key, node); - parent.insertChild(node); - } - - private Node resolveParent(Key key, Node inserted) { - var path = key.split(); - if (path.length == 1) { - return root; - } - var parentPath = Arrays.copyOf(path, path.length - 1); - return resolveParent(root, parentPath, 0, inserted); - } - - private Node resolveParent(Node target, String[] key, int depth, Node inserted) { - if (depth >= key.length) { - return target; - } - for (var child : target.children()) { - if (child.name().equals(key[depth])) { - return resolveParent(child, key, depth + 1, inserted); - } - } - return insertRemainingPath(target, key, depth, inserted.access()); - } - - private Node insertRemainingPath(Node target, String[] key, int depth, Access access) { - for (int index = depth; index < key.length; index++) { - var intermediate = new Node.Path(key[index], access, new ArrayList<>()); - target.insertChild(intermediate); - target = intermediate; - } - return target; - } - - private Node createNode(Key key, Access access) { - if (isMethodSignature(key.value())) { - var signature = MethodSignature.parse(key.value()); - return new Node.Method(signature, access); - } - return new Node.Path(key.lastPart(), access, new ArrayList<>()); - } - - private static boolean isMethodSignature(String path) { - return path.indexOf(MethodSignature.methodSeparator()) >= 0; - } - - // TODO: Create defensive copies after building - public AccessGraph create() { - return new AccessGraph(root); - } - } -} diff --git a/runtime/src/main/java/jsheets/sandbox/SandboxBytecodeCheck.java b/runtime/src/main/java/jsheets/sandbox/SandboxBytecodeCheck.java index 2adbdc8..ce57a6a 100644 --- a/runtime/src/main/java/jsheets/sandbox/SandboxBytecodeCheck.java +++ b/runtime/src/main/java/jsheets/sandbox/SandboxBytecodeCheck.java @@ -30,16 +30,22 @@ private SandboxBytecodeCheck(Collection rules) { public void run(Analysis analysis, byte[] classCode) { var reader = new ClassReader(classCode); - reader.accept(new ClassCheck(rules, analysis), 0); + reader.accept(new ClassCheck(reader.getClassName(), rules, analysis), 0); } static final class ClassCheck extends ClassVisitor { + private final String className; private final Collection rules; private final Analysis analysis; - private ClassCheck(Collection rules, Analysis analysis) { + private ClassCheck( + String className, + Collection rules, + Analysis analysis + ) { super(Opcodes.ASM9); this.rules = rules; + this.className = className; this.analysis = analysis; } @@ -51,23 +57,26 @@ public MethodVisitor visitMethod( String signature, String[] exceptions ) { - return new MethodCheck(name, rules, analysis); + return new MethodCheck(name, className, rules, analysis); } } static final class MethodCheck extends MethodVisitor { private final String name; + private final String className; private final Collection rules; private final Analysis analysis; private MethodCheck( String name, + String className, Collection rules, Analysis analysis ) { super(Opcodes.ASM9); this.name = name; this.rules = rules; + this.className = className; this.analysis = analysis; } @@ -79,10 +88,27 @@ public void visitMethodInsn( String descriptor, boolean isInterface ) { - var call = new Rule.MethodCall(owner, name); + var call = new Rule.MethodCall(createAccessPoint(), owner, name); for (var rule : rules) { rule.visitCall(analysis, call); } } + + @Override + public void visitFieldInsn( + int opcode, + String owner, + String field, + String descriptor + ) { + var access = new Rule.FieldAccess(createAccessPoint(), owner, field); + for (var rule : rules) { + rule.visitFieldAccess(analysis, access); + } + } + + private Rule.AccessPoint createAccessPoint() { + return new Rule.AccessPoint(className, name); + } } } \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/sandbox/SandboxLoader.java b/runtime/src/main/java/jsheets/sandbox/SandboxLoader.java index f309f4d..fb25fa7 100644 --- a/runtime/src/main/java/jsheets/sandbox/SandboxLoader.java +++ b/runtime/src/main/java/jsheets/sandbox/SandboxLoader.java @@ -15,6 +15,7 @@ import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.Enumeration; @@ -22,22 +23,26 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import jdk.jshell.execution.LoaderDelegate; import jdk.jshell.spi.ExecutionControl; import jsheets.sandbox.validation.Analysis; -import jsheets.sandbox.validation.ForbiddenMethodFilter; +import jsheets.sandbox.validation.Rule; public final class SandboxLoader implements LoaderDelegate { - public static SandboxLoader create() { - return new SandboxLoader(); + public static SandboxLoader create(Collection rules) { + Objects.requireNonNull(rules, "rules"); + return new SandboxLoader(rules); } private final SandboxLoader.RemoteClassLoader loader; private final Map> types = new HashMap<>(); + private final Collection rules; - private SandboxLoader() { + private SandboxLoader(Collection rules) { this.loader = new RemoteClassLoader(); + this.rules = rules; } public void install() { @@ -56,7 +61,20 @@ private void loadBinaries(ExecutionControl.ClassBytecodes[] binaries) throws ExecutionControl.ClassInstallException { var analysis = Analysis.create(); - var check = SandboxBytecodeCheck.withRules(ForbiddenMethodFilter.create()); + var check = SandboxBytecodeCheck.withRules(rules); + loadBinariesWithAnalysis(analysis, check, binaries); + checkAnalysisReport(analysis); + } + + private void checkAnalysisReport(Analysis analysis) { + analysis.reportViolations(); + } + + private void loadBinariesWithAnalysis( + Analysis analysis, + SandboxBytecodeCheck check, + ExecutionControl.ClassBytecodes[] binaries + ) throws ExecutionControl.ClassInstallException{ try { for (var binary : binaries) { check.run(analysis, binary.bytecodes()); diff --git a/runtime/src/main/java/jsheets/sandbox/SandboxedEnvironment.java b/runtime/src/main/java/jsheets/sandbox/SandboxedEnvironment.java index c5b1667..7836f5f 100644 --- a/runtime/src/main/java/jsheets/sandbox/SandboxedEnvironment.java +++ b/runtime/src/main/java/jsheets/sandbox/SandboxedEnvironment.java @@ -4,15 +4,23 @@ import jdk.jshell.spi.ExecutionControl; import jdk.jshell.spi.ExecutionControlProvider; import jdk.jshell.spi.ExecutionEnv; +import jsheets.sandbox.validation.Rule; +import java.util.Collection; import java.util.Map; +import java.util.Objects; public final class SandboxedEnvironment implements ExecutionControlProvider { - public static SandboxedEnvironment create() { - return new SandboxedEnvironment(); + public static SandboxedEnvironment create(Collection rules) { + Objects.requireNonNull(rules); + return new SandboxedEnvironment(rules); } - private SandboxedEnvironment() {} + private final Collection rules; + + private SandboxedEnvironment(Collection rules) { + this.rules = rules; + } @Override public String name() { @@ -24,6 +32,6 @@ public ExecutionControl generate( ExecutionEnv environment, Map parameters ) { - return new DirectExecutionControl(SandboxLoader.create()); + return new DirectExecutionControl(SandboxLoader.create(rules)); } } diff --git a/runtime/src/main/java/jsheets/sandbox/access/Access.java b/runtime/src/main/java/jsheets/sandbox/access/Access.java new file mode 100644 index 0000000..22bd77a --- /dev/null +++ b/runtime/src/main/java/jsheets/sandbox/access/Access.java @@ -0,0 +1,7 @@ +package jsheets.sandbox.access; + +public enum Access { + NotSet, + Permitted, + Denied +} diff --git a/runtime/src/main/java/jsheets/sandbox/access/AccessGraph.java b/runtime/src/main/java/jsheets/sandbox/access/AccessGraph.java new file mode 100644 index 0000000..d5ed47f --- /dev/null +++ b/runtime/src/main/java/jsheets/sandbox/access/AccessGraph.java @@ -0,0 +1,154 @@ +package jsheets.sandbox.access; + +import java.util.Collection; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; + +public final class AccessGraph { + private static final AccessGraph empty = new AccessGraph( + AccessGraphNode.Path.create("root", Access.Denied, List.of())); + + public static AccessGraph empty() { + return empty; + } + + public static AccessGraph of(String... permissions) { + var builder = newBuilder(); + for (var permission : permissions) { + if (permission.startsWith("!")) { + var remaining = permission.substring(1).trim(); + builder.deny(AccessKey.infer(remaining)); + } else { + builder.permit(AccessKey.infer(permission.trim())); + } + } + return builder.create(); + } + + public static AccessGraphBuilder newBuilder() { + return new AccessGraphBuilder(); + } + + private final AccessGraphNode root; + + AccessGraph(AccessGraphNode root) { + this.root = root; + } + + private Collection findClosestMatch(AccessKey key) { + return findClosestMatch(root, key.split(), 0); + } + + private static Collection findClosestMatch( + AccessGraphNode node, + String[] key, + int depth + ) { + if (depth >= key.length) { + return List.of(node); + } + if (depth == key.length - 1) { + return findClosestMatchOrMethods(node, key[key.length - 1]); + } + return findClosestChild(node, key, depth); + } + + private static Collection findClosestChild( + AccessGraphNode node, + String[] key, + int depth + ) { + for (var child : node) { + if (child.key().equals(key[depth])) { + return findClosestMatch(child, key, depth + 1); + } + } + return List.of(node); + } + + private static Collection findClosestMatchOrMethods( + AccessGraphNode node, + String lastKey + ) { + var children = node.children() + .filter(child -> child.key().equals(lastKey)) + .toList(); + return children.isEmpty() ? List.of(node) : children; + } + + /** + * Checks if the use of the class or method under the given key is permitted. + *

+ * When checking access to methods prefer + * {@link AccessGraph#isMethodPermitted(MethodSignature)}. + * + * @param key Key of the checked package, class or method. + * @return True if the caller has permission to use classes or methods + * under this key. + */ + public boolean isPermitted(AccessKey key) { + var matches = findClosestMatch(key); + if (matches.isEmpty()) { + return false; + } + var first = matches.iterator().next(); + return first.access().equals(Access.Permitted); + } + + /** + * Checks if a call to the method is permitted. + *

+ * This call is better for checking access to methods because it supports + * overloaded methods. The {@link AccessGraph#isPermitted} method does + * not handle ambiguous calls well. + *

+ * Note, that this method does not support covariant return types. + * + * @param signature Signature of the method that is checked. + * @return True if the caller has permissions to call this method. + */ + public boolean isMethodPermitted(MethodSignature signature) { + var key = AccessKey.slashSeparated(signature.formatWithoutTypes()); + var matches = findClosestMatch(key); + var access = switch (matches.size()) { + case 0 -> Access.Denied; + case 1 -> matches.iterator().next().access(); + default -> findBestMatch(signature, matches).map(AccessGraphNode::access); + }; + return access.equals(Access.Permitted); + } + + private Optional findBestMatch( + MethodSignature signature, + Collection nodes + ) { + for (var node : nodes) { + if (node.matchesMethod(signature)) { + return Optional.of(node); + } + } + return Optional.empty(); + } + + @Override + public String toString() { + return "AccessGraph(%s)".formatted(root); + } + + @Override + public boolean equals(Object target) { + if (this == target) { + return true; + } + return target instanceof AccessGraph graph && ( + Objects.equals(graph.root, root) + ); + } + + @Override + public int hashCode() { + return Objects.hash(root); + } +} \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/sandbox/access/AccessGraphBuilder.java b/runtime/src/main/java/jsheets/sandbox/access/AccessGraphBuilder.java new file mode 100644 index 0000000..6db8918 --- /dev/null +++ b/runtime/src/main/java/jsheets/sandbox/access/AccessGraphBuilder.java @@ -0,0 +1,90 @@ +package jsheets.sandbox.access; + +import java.util.ArrayList; +import java.util.Arrays; + +import com.google.errorprone.annotations.CanIgnoreReturnValue; + +public final class AccessGraphBuilder { + private final AccessGraphNode root = AccessGraphNode.Path.create( + "root", + Access.NotSet, + new ArrayList<>() + ); + + AccessGraphBuilder() {} + + @CanIgnoreReturnValue + public AccessGraphBuilder permit(AccessKey key) { + var node = createNode(key, Access.Permitted); + insert(key, node); + return this; + } + + @CanIgnoreReturnValue + public AccessGraphBuilder deny(AccessKey key) { + var node = createNode(key, Access.Denied); + insert(key, node); + return this; + } + + private void insert(AccessKey key, AccessGraphNode node) { + var parent = resolveParent(key, node); + parent.insertChild(node); + } + + private AccessGraphNode resolveParent( + AccessKey key, + AccessGraphNode inserted + ) { + var path = key.split(); + if (path.length == 1) { + return root; + } + var parentPath = Arrays.copyOf(path, path.length - 1); + return resolveParent(root, parentPath, 0, inserted); + } + + private AccessGraphNode resolveParent( + AccessGraphNode target, + String[] key, + int depth, + AccessGraphNode inserted + ) { + if (depth >= key.length) { + return target; + } + for (var child : target) { + if (child.matchesKey(key[depth])) { + return resolveParent(child, key, depth + 1, inserted); + } + } + return insertRemainingPath(target, key, depth); + } + + private AccessGraphNode insertRemainingPath(AccessGraphNode target, String[] key, int depth) { + for (int index = depth; index < key.length; index++) { + var intermediate = AccessGraphNode.Path.create(key[index], Access.NotSet, new ArrayList<>()); + target.insertChild(intermediate); + target = intermediate; + } + return target; + } + + private AccessGraphNode createNode(AccessKey key, Access access) { + if (isMethodSignature(key.value())) { + var signature = MethodSignature.parse(key.value()); + return AccessGraphNode.Method.create(signature, access, new ArrayList<>()); + } + return AccessGraphNode.Path.create(key.lastPart(), access, new ArrayList<>()); + } + + private static boolean isMethodSignature(String path) { + return path.indexOf(MethodSignature.methodSeparator()) >= 0; + } + + // TODO: Create defensive copies after building + public AccessGraph create() { + return new AccessGraph(root); + } +} \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/sandbox/access/AccessGraphNode.java b/runtime/src/main/java/jsheets/sandbox/access/AccessGraphNode.java new file mode 100644 index 0000000..559f991 --- /dev/null +++ b/runtime/src/main/java/jsheets/sandbox/access/AccessGraphNode.java @@ -0,0 +1,137 @@ +package jsheets.sandbox.access; + +import java.util.Collection; +import java.util.Iterator; +import java.util.Objects; +import java.util.stream.Stream; + +public abstract class AccessGraphNode implements Iterable { + private Access access; + private final Collection children; + + protected AccessGraphNode(Access access, Collection nodes) { + this.access = access; + this.children = nodes; + } + + public abstract String key(); + + public abstract boolean matchesKey(String key); + public abstract boolean matchesMethod(MethodSignature signature); + + public void changeAccess(Access access) { + Objects.requireNonNull(access, "access"); + this.access = access; + } + + public Access access() { + return access; + } + + public void insertChild(AccessGraphNode node) { + Objects.requireNonNull(node, "node"); + this.children.add(node); + } + + public Stream children() { + return children.stream(); + } + + @Override + public Iterator iterator() { + return children.iterator(); + } + + @Override + public String toString() { + return isLeaf() ? formatLeaf() : formatBranch(); + } + + private boolean isLeaf() { + return children.isEmpty(); + } + + private String formatBranch() { + return "AccessGraphNode(key=%s, access=%s, nodes=%s)" + .formatted(key(), access, children); + } + + private String formatLeaf() { + return "AccessGraphNode(key=%s, access=%s)" + .formatted(key(), access); + } + + public static final class Path extends AccessGraphNode { + public static AccessGraphNode create( + String key, + Access access, + Collection nodes + ) { + Objects.requireNonNull(key, "key"); + Objects.requireNonNull(access, "access"); + Objects.requireNonNull(nodes, "nodes"); + return new Path(key, access, nodes); + } + + private final String key; + + private Path(String key, Access access, Collection nodes) { + super(access, nodes); + this.key = key; + } + + @Override + public String key() { + return key; + } + + @Override + public boolean matchesKey(String key) { + return this.key.equals(key); + } + + @Override + public boolean matchesMethod(MethodSignature signature) { + return signature.methodName().equals(key); + } + } + + public static final class Method extends AccessGraphNode { + public static AccessGraphNode create( + MethodSignature method, + Access access, + Collection nodes + ) { + Objects.requireNonNull(method, "method"); + Objects.requireNonNull(access, "access"); + Objects.requireNonNull(nodes, "nodes"); + return new Method(method, access, nodes); + } + + private final MethodSignature method; + + private Method( + MethodSignature method, + Access access, + Collection nodes + ) { + super(access, nodes); + this.method = method; + } + + @Override + public String key() { + return method.formatNameAndParameters(); + } + + @Override + public boolean matchesKey(String key) { + return false; + } + + @Override + public boolean matchesMethod(MethodSignature signature) { + return signature.matches(method); + } + } +} \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/sandbox/access/AccessKey.java b/runtime/src/main/java/jsheets/sandbox/access/AccessKey.java new file mode 100644 index 0000000..b8a27cb --- /dev/null +++ b/runtime/src/main/java/jsheets/sandbox/access/AccessKey.java @@ -0,0 +1,38 @@ +package jsheets.sandbox.access; + +import java.util.Objects; +import java.util.regex.Pattern; + +public record AccessKey(Pattern separator, String value) { + public static AccessKey infer(String value) { + return value.contains("/") ? slashSeparated(value) : dotSeparated(value); + } + + private static final Pattern dotOrMethodSeparator = + Pattern.compile("[.#]"); + + public static AccessKey dotSeparated(String value) { + Objects.requireNonNull(value, "value"); + return new AccessKey(dotOrMethodSeparator, value); + } + + private static final Pattern slashOrMethodSeparator = + Pattern.compile("[/#]"); + + public static AccessKey slashSeparated(String value) { + Objects.requireNonNull(value, "value"); + return new AccessKey(slashOrMethodSeparator, value); + } + + public String[] split() { + return separator.split(value); + } + + public String lastPart() { + var parts = split(); + if (parts.length == 0) { + return value; + } + return parts[parts.length - 1]; + } +} \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/sandbox/MethodSignature.java b/runtime/src/main/java/jsheets/sandbox/access/MethodSignature.java similarity index 90% rename from runtime/src/main/java/jsheets/sandbox/MethodSignature.java rename to runtime/src/main/java/jsheets/sandbox/access/MethodSignature.java index ba1fc6d..c77b5a1 100644 --- a/runtime/src/main/java/jsheets/sandbox/MethodSignature.java +++ b/runtime/src/main/java/jsheets/sandbox/access/MethodSignature.java @@ -1,14 +1,13 @@ -package jsheets.sandbox; +package jsheets.sandbox.access; import io.soabase.recordbuilder.core.RecordBuilder; -import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.regex.Pattern; @RecordBuilder -record MethodSignature( +public record MethodSignature( String className, String methodName, String returnType, @@ -70,14 +69,17 @@ public boolean matches(MethodSignature signature) { } public String format() { - return "%s %s#%s(%s)".formatted( + return "%s %s#%s".formatted( returnType, className, - methodName, - Arrays.toString(parameterTypes.toArray()) + formatNameAndParameters() ); } + public String formatNameAndParameters() { + return "%s(%s)".formatted(methodName, String.join(", ", parameterTypes)); + } + public String formatWithoutTypes() { return "%s#%s".formatted(className, methodName); } diff --git a/runtime/src/main/java/jsheets/sandbox/validation/Analysis.java b/runtime/src/main/java/jsheets/sandbox/validation/Analysis.java index 4b405ee..93c6c56 100644 --- a/runtime/src/main/java/jsheets/sandbox/validation/Analysis.java +++ b/runtime/src/main/java/jsheets/sandbox/validation/Analysis.java @@ -1,11 +1,36 @@ package jsheets.sandbox.validation; +import java.util.Collection; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.stream.Collectors; + public final class Analysis { public static Analysis create() { return new Analysis(); } - public record Violation() {} + public interface Violation {} + + private final Collection violations = + new ConcurrentLinkedQueue<>(); private Analysis() {} + + public void report(Violation violation) { + violations.add(violation); + } + + public void reportViolations() { + switch (violations.size()) { + case 0 -> { } + case 1 -> throw new RuntimeException( + violations.iterator().next().toString() + ); + default -> throw new RuntimeException( + violations.stream() + .map(Violation::toString) + .collect(Collectors.joining(", ")) + ); + } + } } \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/sandbox/validation/ForbiddenMemberFilter.java b/runtime/src/main/java/jsheets/sandbox/validation/ForbiddenMemberFilter.java new file mode 100644 index 0000000..b3ec424 --- /dev/null +++ b/runtime/src/main/java/jsheets/sandbox/validation/ForbiddenMemberFilter.java @@ -0,0 +1,48 @@ +package jsheets.sandbox.validation; + +import java.util.Objects; + +import jsheets.sandbox.access.AccessGraph; +import jsheets.sandbox.access.AccessKey; +import jsheets.sandbox.access.MethodSignature; +import jsheets.sandbox.access.MethodSignatureBuilder; + +public final class ForbiddenMemberFilter implements Rule { + public static ForbiddenMemberFilter create(AccessGraph accessGraph) { + Objects.requireNonNull(accessGraph, "accessGraph"); + return new ForbiddenMemberFilter(accessGraph); + } + + private final AccessGraph accessGraph; + + private ForbiddenMemberFilter(AccessGraph accessGraph) { + this.accessGraph = accessGraph; + } + + record ForbiddenMethod(MethodSignature method) implements Analysis.Violation {} + + @Override + public void visitCall(Analysis analysis, MethodCall call) { + var signature = createSignatureOfCall(call); + if (!accessGraph.isMethodPermitted(signature)) { + analysis.report(new ForbiddenMethod(signature)); + } + } + + private MethodSignature createSignatureOfCall(MethodCall call) { + return MethodSignatureBuilder.builder() + .className(call.owner()) + .methodName(call.method()) + .build(); + } + + record ForbiddenField(String owner, String field) implements Analysis.Violation {} + + @Override + public void visitFieldAccess(Analysis analysis, FieldAccess access) { + var key = "%s/%s".formatted(access.owner(), access.field()); + if (!accessGraph.isPermitted(AccessKey.slashSeparated(key))) { + analysis.report(new ForbiddenField(access.owner(), access.field())); + } + } +} \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/sandbox/validation/ForbiddenMethodFilter.java b/runtime/src/main/java/jsheets/sandbox/validation/ForbiddenMethodFilter.java deleted file mode 100644 index 3196bd1..0000000 --- a/runtime/src/main/java/jsheets/sandbox/validation/ForbiddenMethodFilter.java +++ /dev/null @@ -1,14 +0,0 @@ -package jsheets.sandbox.validation; - -public final class ForbiddenMethodFilter implements Rule { - public static ForbiddenMethodFilter create() { - return new ForbiddenMethodFilter(); - } - - private ForbiddenMethodFilter() {} - - @Override - public void visitCall(Analysis analysis, MethodCall call) { - System.out.println(call); - } -} \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/sandbox/validation/Rule.java b/runtime/src/main/java/jsheets/sandbox/validation/Rule.java index 698e0b6..0a07d04 100644 --- a/runtime/src/main/java/jsheets/sandbox/validation/Rule.java +++ b/runtime/src/main/java/jsheets/sandbox/validation/Rule.java @@ -1,7 +1,10 @@ package jsheets.sandbox.validation; public interface Rule { - record MethodCall(String owner, String method) { } + record AccessPoint(String className, String methodName) { } + record MethodCall(AccessPoint accessPoint, String owner, String method) { } + record FieldAccess(AccessPoint accessPoint, String owner, String field) { } default void visitCall(Analysis analysis, MethodCall call) {} + default void visitFieldAccess(Analysis analysis, FieldAccess access) {} } diff --git a/runtime/src/test/java/jsheets/sandbox/AccessGraphTest.java b/runtime/src/test/java/jsheets/sandbox/AccessGraphTest.java index 89d6960..a56d545 100644 --- a/runtime/src/test/java/jsheets/sandbox/AccessGraphTest.java +++ b/runtime/src/test/java/jsheets/sandbox/AccessGraphTest.java @@ -1,6 +1,9 @@ package jsheets.sandbox; import com.google.common.base.Charsets; + +import jsheets.sandbox.access.AccessGraph; +import jsheets.sandbox.access.MethodSignature; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -8,10 +11,9 @@ import java.io.IOException; import java.util.List; -import static jsheets.sandbox.AccessGraph.Key.*; +import static jsheets.sandbox.access.AccessKey.*; public class AccessGraphTest { - @Test public void testReadingFromFile() { var specification = readFileContent("access-graph.txt"); diff --git a/runtime/src/test/java/jsheets/sandbox/MethodSignatureTest.java b/runtime/src/test/java/jsheets/sandbox/MethodSignatureTest.java index a5a6898..b7c261d 100644 --- a/runtime/src/test/java/jsheets/sandbox/MethodSignatureTest.java +++ b/runtime/src/test/java/jsheets/sandbox/MethodSignatureTest.java @@ -1,5 +1,7 @@ package jsheets.sandbox; +import jsheets.sandbox.access.MethodSignature; +import jsheets.sandbox.access.MethodSignatureBuilder; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/runtime/src/test/java/jsheets/sandbox/PermissionGraphTest.java b/runtime/src/test/java/jsheets/sandbox/PermissionGraphTest.java index e53758b..8959b5a 100644 --- a/runtime/src/test/java/jsheets/sandbox/PermissionGraphTest.java +++ b/runtime/src/test/java/jsheets/sandbox/PermissionGraphTest.java @@ -1,18 +1,22 @@ package jsheets.sandbox; +import jsheets.sandbox.access.AccessGraph; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; -import static jsheets.sandbox.AccessGraph.Key.slashSeparated; +import static jsheets.sandbox.access.AccessKey.slashSeparated; public class PermissionGraphTest { + @Test public void testDirectPath() { var graph = AccessGraph.of( "java.lang", "java.util.List", "void java.lang.Thread#sleep()" ); + Assertions.assertTrue(graph.isPermitted(slashSeparated("java/io"))); Assertions.assertTrue(graph.isPermitted(slashSeparated("java/lang"))); - Assertions.assertTrue(graph.isPermitted(slashSeparated("java/lang"))); - Assertions.assertTrue(graph.isPermitted(slashSeparated("java/lang"))); + Assertions.assertTrue(graph.isPermitted(slashSeparated("java/util/List"))); + Assertions.assertTrue(graph.isPermitted(slashSeparated("java/lang/Thread#sleep()"))); } } diff --git a/runtime/src/test/java/jsheets/sandbox/SandboxExecutionTest.java b/runtime/src/test/java/jsheets/sandbox/SandboxExecutionTest.java index c2d77e5..0102061 100644 --- a/runtime/src/test/java/jsheets/sandbox/SandboxExecutionTest.java +++ b/runtime/src/test/java/jsheets/sandbox/SandboxExecutionTest.java @@ -1,22 +1,36 @@ package jsheets.sandbox; import jdk.jshell.JShell; +import jsheets.sandbox.access.AccessGraph; +import jsheets.sandbox.validation.ForbiddenMemberFilter; import org.junit.jupiter.api.Test; +import java.util.List; import java.util.Map; public final class SandboxExecutionTest { @Test public void test() { var shell = createSandboxedShell(); - shell.eval("System.out.println(\"Hello, World!\");"); + shell.eval(""" + System.out.println("Hello, World!"); + System.err.println("Hello, World!"); + """); } private JShell createSandboxedShell() { + var accessGraph = AccessGraph.of( + "java.lang.Object", + "java.lang.System.out", + "java.io.PrintStream#println" + ); + System.out.println(accessGraph); return JShell.builder() .out(System.out) .err(System.err) - .executionEngine(SandboxedEnvironment.create(), Map.of()) - .build(); + .executionEngine( + SandboxedEnvironment.create(List.of(ForbiddenMemberFilter.create(accessGraph))), + Map.of() + ).build(); } } From d16d92d47878f012963789a593973ead630b1401 Mon Sep 17 00:00:00 2001 From: Merlin Osayimwen Date: Fri, 15 Oct 2021 00:58:34 +0200 Subject: [PATCH 04/14] Add runtime application --- build.gradle | 12 +- evaluation/build.gradle | 34 +++++ .../java/jsheets}/evaluation/Evaluation.java | 2 +- .../jsheets}/evaluation/EvaluationEngine.java | 2 +- .../sandbox/SandboxBytecodeCheck.java | 14 +- .../evaluation}/sandbox/SandboxConfig.java | 2 +- .../evaluation}/sandbox/SandboxLoader.java | 8 +- .../sandbox/SandboxedEnvironment.java | 4 +- .../evaluation}/sandbox/access/Access.java | 2 +- .../sandbox/access/AccessGraph.java | 23 ++-- .../sandbox/access/AccessGraphBuilder.java | 34 ++--- .../sandbox/access/AccessGraphNode.java | 27 +++- .../evaluation}/sandbox/access/AccessKey.java | 2 +- .../sandbox/access/MethodSignature.java | 73 ++++++---- .../sandbox/validation/Analysis.java | 57 ++++++++ .../validation/ForbiddenMemberFilter.java | 27 ++-- .../evaluation/sandbox/validation/Rule.java | 23 ++++ .../jsheets}/evaluation/shell/MessageLog.java | 4 +- .../evaluation/shell/MessageOutput.java | 4 +- .../evaluation/shell/ShellEvaluation.java | 8 +- .../shell/ShellEvaluationEngine.java | 14 +- .../environment/ExecutionEnvironment.java | 2 +- .../environment/StandardEnvironment.java | 2 +- .../inprocess/EmbeddedEnvironment.java | 4 +- .../inprocess/InProcessExecutionControl.java | 2 +- .../environment/inprocess/MultiTenancy.java | 2 +- .../environment/inprocess/Preemption.java | 2 +- .../shell/environment/inprocess/Tenancy.java | 2 +- .../shell/execution/DirectExecution.java | 2 +- .../shell/execution/ExecutionMethod.java | 2 +- .../shell/execution/ExhaustiveExecution.java | 2 +- .../SystemBasedExecutionMethodFactory.java | 2 +- .../java/jsheets/output/CapturingOutput.java | 0 .../jsheets/output/ListeningPrintStream.java | 0 .../jsheets/output/TenantBasedOutput.java | 0 .../java/jsheets/source/SharedSources.java | 0 .../src/main/resources/.sandbox | 0 .../sandbox/SandboxExecutionTest.java | 50 +++++++ .../sandbox/access}/AccessGraphTest.java | 37 +++-- .../sandbox/access}/MethodSignatureTest.java | 14 +- .../shell/ExhaustiveExecutionTest.java | 4 +- .../shell/ShellEvaluationEngineTest.java | 6 +- .../jsheets/output/TenantBasedOutputTest.java | 0 .../src/test/resources/access-graph.txt | 0 protocol/build.gradle | 3 + runtime/build.gradle | 82 +++++++++-- runtime/deploy/Dockerfile | 47 +++++++ runtime/deploy/entrypoint.sh | 17 +++ .../main/java/jsheets/config/CamelCase.java | 55 ++++++++ .../java/jsheets/config/CombinedConfig.java | 43 ++++++ .../src/main/java/jsheets/config/Config.java | 93 +++++++++++++ .../jsheets/config/EnvironmentConfig.java | 44 ++++++ .../java/jsheets/config/MissingField.java | 26 ++++ .../main/java/jsheets/config/RawConfig.java | 63 +++++++++ .../java/jsheets/config/ResolvedField.java | 25 ++++ .../config/consul/ConsulConfigSource.java | 50 +++++++ .../src/main/java/jsheets/runtime/App.java | 53 +++++++ .../java/jsheets/runtime/ConfigModule.java | 45 ++++++ .../java/jsheets/runtime/ConsulModule.java | 58 ++++++++ .../java/jsheets/runtime/ServerSetup.java | 129 ++++++++++++++++++ .../jsheets/runtime/ServerSetupModule.java | 96 +++++++++++++ .../runtime/SnippetRuntimeService.java | 109 +++++++++++++++ .../runtime/discovery/AdvertisementHook.java | 51 +++++++ .../ConsulServiceAdvertisementChannel.java | 82 +++++++++++ .../discovery/ServiceAdvertisement.java | 11 ++ .../ServiceAdvertisementChannel.java | 7 + .../runtime/evaluation/EvaluationModule.java | 53 +++++++ .../evaluation/SandboxConfigSource.java | 69 ++++++++++ .../jsheets/sandbox/validation/Analysis.java | 36 ----- .../java/jsheets/sandbox/validation/Rule.java | 10 -- .../evaluation/sandbox/accessGraph.txt | 0 .../java/jsheets/config/CamelCaseTest.java | 53 +++++++ .../jsheets/config/EnvironmentConfigTest.java | 64 +++++++++ .../jsheets/sandbox/PermissionGraphTest.java | 22 --- .../jsheets/sandbox/SandboxExecutionTest.java | 36 ----- server/build.gradle | 6 +- .../evaluation/EvaluationConnection.java | 4 +- .../server/evaluation/EvaluationModule.java | 4 +- settings.gradle | 3 +- 79 files changed, 1800 insertions(+), 260 deletions(-) create mode 100644 evaluation/build.gradle rename {runtime/src/main/java/jsheets/runtime => evaluation/src/main/java/jsheets}/evaluation/Evaluation.java (84%) rename {runtime/src/main/java/jsheets/runtime => evaluation/src/main/java/jsheets}/evaluation/EvaluationEngine.java (81%) rename {runtime/src/main/java/jsheets => evaluation/src/main/java/jsheets/evaluation}/sandbox/SandboxBytecodeCheck.java (85%) rename {runtime/src/main/java/jsheets => evaluation/src/main/java/jsheets/evaluation}/sandbox/SandboxConfig.java (65%) rename {runtime/src/main/java/jsheets => evaluation/src/main/java/jsheets/evaluation}/sandbox/SandboxLoader.java (97%) rename {runtime/src/main/java/jsheets => evaluation/src/main/java/jsheets/evaluation}/sandbox/SandboxedEnvironment.java (90%) rename {runtime/src/main/java/jsheets => evaluation/src/main/java/jsheets/evaluation}/sandbox/access/Access.java (56%) rename {runtime/src/main/java/jsheets => evaluation/src/main/java/jsheets/evaluation}/sandbox/access/AccessGraph.java (83%) rename {runtime/src/main/java/jsheets => evaluation/src/main/java/jsheets/evaluation}/sandbox/access/AccessGraphBuilder.java (73%) rename {runtime/src/main/java/jsheets => evaluation/src/main/java/jsheets/evaluation}/sandbox/access/AccessGraphNode.java (85%) rename {runtime/src/main/java/jsheets => evaluation/src/main/java/jsheets/evaluation}/sandbox/access/AccessKey.java (95%) rename {runtime/src/main/java/jsheets => evaluation/src/main/java/jsheets/evaluation}/sandbox/access/MethodSignature.java (52%) create mode 100644 evaluation/src/main/java/jsheets/evaluation/sandbox/validation/Analysis.java rename {runtime/src/main/java/jsheets => evaluation/src/main/java/jsheets/evaluation}/sandbox/validation/ForbiddenMemberFilter.java (54%) create mode 100644 evaluation/src/main/java/jsheets/evaluation/sandbox/validation/Rule.java rename {runtime/src/main/java/jsheets/runtime => evaluation/src/main/java/jsheets}/evaluation/shell/MessageLog.java (94%) rename {runtime/src/main/java/jsheets/runtime => evaluation/src/main/java/jsheets}/evaluation/shell/MessageOutput.java (95%) rename {runtime/src/main/java/jsheets/runtime => evaluation/src/main/java/jsheets}/evaluation/shell/ShellEvaluation.java (96%) rename {runtime/src/main/java/jsheets/runtime => evaluation/src/main/java/jsheets}/evaluation/shell/ShellEvaluationEngine.java (92%) rename {runtime/src/main/java/jsheets => evaluation/src/main/java/jsheets/evaluation}/shell/environment/ExecutionEnvironment.java (83%) rename {runtime/src/main/java/jsheets => evaluation/src/main/java/jsheets/evaluation}/shell/environment/StandardEnvironment.java (96%) rename {runtime/src/main/java/jsheets => evaluation/src/main/java/jsheets/evaluation}/shell/environment/inprocess/EmbeddedEnvironment.java (93%) rename {runtime/src/main/java/jsheets => evaluation/src/main/java/jsheets/evaluation}/shell/environment/inprocess/InProcessExecutionControl.java (98%) rename {runtime/src/main/java/jsheets => evaluation/src/main/java/jsheets/evaluation}/shell/environment/inprocess/MultiTenancy.java (96%) rename {runtime/src/main/java/jsheets => evaluation/src/main/java/jsheets/evaluation}/shell/environment/inprocess/Preemption.java (75%) rename {runtime/src/main/java/jsheets => evaluation/src/main/java/jsheets/evaluation}/shell/environment/inprocess/Tenancy.java (70%) rename {runtime/src/main/java/jsheets => evaluation/src/main/java/jsheets/evaluation}/shell/execution/DirectExecution.java (93%) rename {runtime/src/main/java/jsheets => evaluation/src/main/java/jsheets/evaluation}/shell/execution/ExecutionMethod.java (84%) rename {runtime/src/main/java/jsheets => evaluation/src/main/java/jsheets/evaluation}/shell/execution/ExhaustiveExecution.java (99%) rename {runtime/src/main/java/jsheets => evaluation/src/main/java/jsheets/evaluation}/shell/execution/SystemBasedExecutionMethodFactory.java (92%) rename {runtime => evaluation}/src/main/java/jsheets/output/CapturingOutput.java (100%) rename {runtime => evaluation}/src/main/java/jsheets/output/ListeningPrintStream.java (100%) rename {runtime => evaluation}/src/main/java/jsheets/output/TenantBasedOutput.java (100%) rename {runtime => evaluation}/src/main/java/jsheets/source/SharedSources.java (100%) rename {runtime => evaluation}/src/main/resources/.sandbox (100%) create mode 100644 evaluation/src/test/java/jsheets/evaluation/sandbox/SandboxExecutionTest.java rename {runtime/src/test/java/jsheets/sandbox => evaluation/src/test/java/jsheets/evaluation/sandbox/access}/AccessGraphTest.java (61%) rename {runtime/src/test/java/jsheets/sandbox => evaluation/src/test/java/jsheets/evaluation/sandbox/access}/MethodSignatureTest.java (60%) rename {runtime/src/test/java/jsheets => evaluation/src/test/java/jsheets/evaluation}/shell/ExhaustiveExecutionTest.java (95%) rename {runtime/src/test/java/jsheets/runtime => evaluation/src/test/java/jsheets}/evaluation/shell/ShellEvaluationEngineTest.java (94%) rename {runtime => evaluation}/src/test/java/jsheets/output/TenantBasedOutputTest.java (100%) rename {runtime => evaluation}/src/test/resources/access-graph.txt (100%) create mode 100644 runtime/deploy/Dockerfile create mode 100644 runtime/deploy/entrypoint.sh create mode 100644 runtime/src/main/java/jsheets/config/CamelCase.java create mode 100644 runtime/src/main/java/jsheets/config/CombinedConfig.java create mode 100644 runtime/src/main/java/jsheets/config/Config.java create mode 100644 runtime/src/main/java/jsheets/config/EnvironmentConfig.java create mode 100644 runtime/src/main/java/jsheets/config/MissingField.java create mode 100644 runtime/src/main/java/jsheets/config/RawConfig.java create mode 100644 runtime/src/main/java/jsheets/config/ResolvedField.java create mode 100644 runtime/src/main/java/jsheets/config/consul/ConsulConfigSource.java create mode 100644 runtime/src/main/java/jsheets/runtime/App.java create mode 100644 runtime/src/main/java/jsheets/runtime/ConfigModule.java create mode 100644 runtime/src/main/java/jsheets/runtime/ConsulModule.java create mode 100644 runtime/src/main/java/jsheets/runtime/ServerSetup.java create mode 100644 runtime/src/main/java/jsheets/runtime/ServerSetupModule.java create mode 100644 runtime/src/main/java/jsheets/runtime/SnippetRuntimeService.java create mode 100644 runtime/src/main/java/jsheets/runtime/discovery/AdvertisementHook.java create mode 100644 runtime/src/main/java/jsheets/runtime/discovery/ConsulServiceAdvertisementChannel.java create mode 100644 runtime/src/main/java/jsheets/runtime/discovery/ServiceAdvertisement.java create mode 100644 runtime/src/main/java/jsheets/runtime/discovery/ServiceAdvertisementChannel.java create mode 100644 runtime/src/main/java/jsheets/runtime/evaluation/EvaluationModule.java create mode 100644 runtime/src/main/java/jsheets/runtime/evaluation/SandboxConfigSource.java delete mode 100644 runtime/src/main/java/jsheets/sandbox/validation/Analysis.java delete mode 100644 runtime/src/main/java/jsheets/sandbox/validation/Rule.java create mode 100644 runtime/src/main/resources/runtime/evaluation/sandbox/accessGraph.txt create mode 100644 runtime/src/test/java/jsheets/config/CamelCaseTest.java create mode 100644 runtime/src/test/java/jsheets/config/EnvironmentConfigTest.java delete mode 100644 runtime/src/test/java/jsheets/sandbox/PermissionGraphTest.java delete mode 100644 runtime/src/test/java/jsheets/sandbox/SandboxExecutionTest.java diff --git a/build.gradle b/build.gradle index 469bb4a..824fce8 100644 --- a/build.gradle +++ b/build.gradle @@ -5,8 +5,12 @@ allprojects { } ext { - floggerVersion = '0.6' - junitPlatformVersion = '5.7.2' - protobufJavaVersion = '3.17.3' - micrometerVersion = 'latest.integration' + grpcVersion = '1.41.0' + floggerVersion = '0.7.1' + junitPlatformVersion = '5.8.1' + protobufJavaVersion = '3.18.1' + micrometerVersion = '1.7.4' + consuleClientVersion = '1.4.5' + javaxAnnotationVersion = '1.3.2' + recordBuilderVersion = '26' } \ No newline at end of file diff --git a/evaluation/build.gradle b/evaluation/build.gradle new file mode 100644 index 0000000..c17c1f7 --- /dev/null +++ b/evaluation/build.gradle @@ -0,0 +1,34 @@ +plugins { + id 'java' +} + +group 'dev.jsheets' +version '0.1.0' + +sourceCompatibility = 16 +targetCompatibility = 16 + +repositories { + mavenCentral() +} + +ext { + asmVersion = '9.2' +} + +dependencies { + implementation project(':protocol') + implementation "org.ow2.asm:asm:$asmVersion" + implementation "com.google.protobuf:protobuf-java-util:$protobufJavaVersion" + implementation "com.google.flogger:flogger:$floggerVersion" + testImplementation "org.junit.jupiter:junit-jupiter-api:$junitPlatformVersion" + testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:$junitPlatformVersion" + testRuntimeOnly "com.google.flogger:flogger-slf4j-backend:$floggerVersion" + compileOnly "io.soabase.record-builder:record-builder-core:$recordBuilderVersion" + annotationProcessor "io.soabase.record-builder:record-builder-processor:$recordBuilderVersion" +} + +test { + useJUnitPlatform() + jvmArgs += ['--add-opens', 'jdk.jshell/jdk.jshell=ALL-UNNAMED'] +} \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/runtime/evaluation/Evaluation.java b/evaluation/src/main/java/jsheets/evaluation/Evaluation.java similarity index 84% rename from runtime/src/main/java/jsheets/runtime/evaluation/Evaluation.java rename to evaluation/src/main/java/jsheets/evaluation/Evaluation.java index b69bed2..f040e3c 100644 --- a/runtime/src/main/java/jsheets/runtime/evaluation/Evaluation.java +++ b/evaluation/src/main/java/jsheets/evaluation/Evaluation.java @@ -1,4 +1,4 @@ -package jsheets.runtime.evaluation; +package jsheets.evaluation; import jsheets.EvaluateResponse; diff --git a/runtime/src/main/java/jsheets/runtime/evaluation/EvaluationEngine.java b/evaluation/src/main/java/jsheets/evaluation/EvaluationEngine.java similarity index 81% rename from runtime/src/main/java/jsheets/runtime/evaluation/EvaluationEngine.java rename to evaluation/src/main/java/jsheets/evaluation/EvaluationEngine.java index 082178d..410f172 100644 --- a/runtime/src/main/java/jsheets/runtime/evaluation/EvaluationEngine.java +++ b/evaluation/src/main/java/jsheets/evaluation/EvaluationEngine.java @@ -1,4 +1,4 @@ -package jsheets.runtime.evaluation; +package jsheets.evaluation; import jsheets.StartEvaluationRequest; diff --git a/runtime/src/main/java/jsheets/sandbox/SandboxBytecodeCheck.java b/evaluation/src/main/java/jsheets/evaluation/sandbox/SandboxBytecodeCheck.java similarity index 85% rename from runtime/src/main/java/jsheets/sandbox/SandboxBytecodeCheck.java rename to evaluation/src/main/java/jsheets/evaluation/sandbox/SandboxBytecodeCheck.java index ce57a6a..8c1b67a 100644 --- a/runtime/src/main/java/jsheets/sandbox/SandboxBytecodeCheck.java +++ b/evaluation/src/main/java/jsheets/evaluation/sandbox/SandboxBytecodeCheck.java @@ -1,15 +1,16 @@ -package jsheets.sandbox; +package jsheets.evaluation.sandbox; import java.util.Collection; import java.util.List; import java.util.Objects; -import jsheets.sandbox.validation.Analysis; -import jsheets.sandbox.validation.Rule; +import jsheets.evaluation.sandbox.validation.Analysis; +import jsheets.evaluation.sandbox.validation.Rule; import org.objectweb.asm.ClassReader; import org.objectweb.asm.ClassVisitor; import org.objectweb.asm.MethodVisitor; import org.objectweb.asm.Opcodes; +import org.objectweb.asm.Type; public final class SandboxBytecodeCheck { public static SandboxBytecodeCheck withRules(Rule... rules) { @@ -88,7 +89,9 @@ public void visitMethodInsn( String descriptor, boolean isInterface ) { - var call = new Rule.MethodCall(createAccessPoint(), owner, name); + var type = Type.getMethodType(descriptor); + var ownerClass = Type.getObjectType(owner).getClassName(); + var call = new Rule.MethodCall(createAccessPoint(), ownerClass, name, type); for (var rule : rules) { rule.visitCall(analysis, call); } @@ -101,7 +104,8 @@ public void visitFieldInsn( String field, String descriptor ) { - var access = new Rule.FieldAccess(createAccessPoint(), owner, field); + var ownerClass = Type.getObjectType(owner).getClassName(); + var access = new Rule.FieldAccess(createAccessPoint(), ownerClass, field); for (var rule : rules) { rule.visitFieldAccess(analysis, access); } diff --git a/runtime/src/main/java/jsheets/sandbox/SandboxConfig.java b/evaluation/src/main/java/jsheets/evaluation/sandbox/SandboxConfig.java similarity index 65% rename from runtime/src/main/java/jsheets/sandbox/SandboxConfig.java rename to evaluation/src/main/java/jsheets/evaluation/sandbox/SandboxConfig.java index 3dadad9..916bd20 100644 --- a/runtime/src/main/java/jsheets/sandbox/SandboxConfig.java +++ b/evaluation/src/main/java/jsheets/evaluation/sandbox/SandboxConfig.java @@ -1,4 +1,4 @@ -package jsheets.sandbox; +package jsheets.evaluation.sandbox; public final class SandboxConfig { private SandboxConfig() {} diff --git a/runtime/src/main/java/jsheets/sandbox/SandboxLoader.java b/evaluation/src/main/java/jsheets/evaluation/sandbox/SandboxLoader.java similarity index 97% rename from runtime/src/main/java/jsheets/sandbox/SandboxLoader.java rename to evaluation/src/main/java/jsheets/evaluation/sandbox/SandboxLoader.java index fb25fa7..6957bf6 100644 --- a/runtime/src/main/java/jsheets/sandbox/SandboxLoader.java +++ b/evaluation/src/main/java/jsheets/evaluation/sandbox/SandboxLoader.java @@ -1,4 +1,4 @@ -package jsheets.sandbox; +package jsheets.evaluation.sandbox; import java.io.ByteArrayInputStream; import java.io.File; @@ -27,10 +27,12 @@ import jdk.jshell.execution.LoaderDelegate; import jdk.jshell.spi.ExecutionControl; -import jsheets.sandbox.validation.Analysis; -import jsheets.sandbox.validation.Rule; +import jsheets.evaluation.sandbox.validation.Analysis; +import jsheets.evaluation.sandbox.validation.Rule; public final class SandboxLoader implements LoaderDelegate { + // This class is based on Java's loader for DirectExecution. + public static SandboxLoader create(Collection rules) { Objects.requireNonNull(rules, "rules"); return new SandboxLoader(rules); diff --git a/runtime/src/main/java/jsheets/sandbox/SandboxedEnvironment.java b/evaluation/src/main/java/jsheets/evaluation/sandbox/SandboxedEnvironment.java similarity index 90% rename from runtime/src/main/java/jsheets/sandbox/SandboxedEnvironment.java rename to evaluation/src/main/java/jsheets/evaluation/sandbox/SandboxedEnvironment.java index 7836f5f..8830ebd 100644 --- a/runtime/src/main/java/jsheets/sandbox/SandboxedEnvironment.java +++ b/evaluation/src/main/java/jsheets/evaluation/sandbox/SandboxedEnvironment.java @@ -1,10 +1,10 @@ -package jsheets.sandbox; +package jsheets.evaluation.sandbox; import jdk.jshell.execution.DirectExecutionControl; import jdk.jshell.spi.ExecutionControl; import jdk.jshell.spi.ExecutionControlProvider; import jdk.jshell.spi.ExecutionEnv; -import jsheets.sandbox.validation.Rule; +import jsheets.evaluation.sandbox.validation.Rule; import java.util.Collection; import java.util.Map; diff --git a/runtime/src/main/java/jsheets/sandbox/access/Access.java b/evaluation/src/main/java/jsheets/evaluation/sandbox/access/Access.java similarity index 56% rename from runtime/src/main/java/jsheets/sandbox/access/Access.java rename to evaluation/src/main/java/jsheets/evaluation/sandbox/access/Access.java index 22bd77a..61f7990 100644 --- a/runtime/src/main/java/jsheets/sandbox/access/Access.java +++ b/evaluation/src/main/java/jsheets/evaluation/sandbox/access/Access.java @@ -1,4 +1,4 @@ -package jsheets.sandbox.access; +package jsheets.evaluation.sandbox.access; public enum Access { NotSet, diff --git a/runtime/src/main/java/jsheets/sandbox/access/AccessGraph.java b/evaluation/src/main/java/jsheets/evaluation/sandbox/access/AccessGraph.java similarity index 83% rename from runtime/src/main/java/jsheets/sandbox/access/AccessGraph.java rename to evaluation/src/main/java/jsheets/evaluation/sandbox/access/AccessGraph.java index d5ed47f..59301a3 100644 --- a/runtime/src/main/java/jsheets/sandbox/access/AccessGraph.java +++ b/evaluation/src/main/java/jsheets/evaluation/sandbox/access/AccessGraph.java @@ -1,10 +1,9 @@ -package jsheets.sandbox.access; +package jsheets.evaluation.sandbox.access; import java.util.Collection; import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.stream.Collectors; public final class AccessGraph { private static final AccessGraph empty = new AccessGraph( @@ -50,7 +49,7 @@ private static Collection findClosestMatch( return List.of(node); } if (depth == key.length - 1) { - return findClosestMatchOrMethods(node, key[key.length - 1]); + return listClosestChildrenOrParent(node, key[key.length - 1]); } return findClosestChild(node, key, depth); } @@ -61,19 +60,19 @@ private static Collection findClosestChild( int depth ) { for (var child : node) { - if (child.key().equals(key[depth])) { + if (child.matchesKey(key[depth])) { return findClosestMatch(child, key, depth + 1); } } return List.of(node); } - private static Collection findClosestMatchOrMethods( + private static Collection listClosestChildrenOrParent( AccessGraphNode node, String lastKey ) { var children = node.children() - .filter(child -> child.key().equals(lastKey)) + .filter(child -> child.matchesKey(lastKey)) .toList(); return children.isEmpty() ? List.of(node) : children; } @@ -110,17 +109,21 @@ public boolean isPermitted(AccessKey key) { * @return True if the caller has permissions to call this method. */ public boolean isMethodPermitted(MethodSignature signature) { - var key = AccessKey.slashSeparated(signature.formatWithoutTypes()); + var key = AccessKey.infer(signature.formatWithoutTypes()); var matches = findClosestMatch(key); var access = switch (matches.size()) { case 0 -> Access.Denied; - case 1 -> matches.iterator().next().access(); - default -> findBestMatch(signature, matches).map(AccessGraphNode::access); + case 1 -> { + var result = matches.iterator().next(); + boolean isMatch = !result.isClassMember() || result.matchesMethod(signature); + yield isMatch ? result.access() : Access.NotSet; + } + default -> findBestMethodMatch(signature, matches).map(AccessGraphNode::access); }; return access.equals(Access.Permitted); } - private Optional findBestMatch( + private Optional findBestMethodMatch( MethodSignature signature, Collection nodes ) { diff --git a/runtime/src/main/java/jsheets/sandbox/access/AccessGraphBuilder.java b/evaluation/src/main/java/jsheets/evaluation/sandbox/access/AccessGraphBuilder.java similarity index 73% rename from runtime/src/main/java/jsheets/sandbox/access/AccessGraphBuilder.java rename to evaluation/src/main/java/jsheets/evaluation/sandbox/access/AccessGraphBuilder.java index 6db8918..959fb7c 100644 --- a/runtime/src/main/java/jsheets/sandbox/access/AccessGraphBuilder.java +++ b/evaluation/src/main/java/jsheets/evaluation/sandbox/access/AccessGraphBuilder.java @@ -1,4 +1,4 @@ -package jsheets.sandbox.access; +package jsheets.evaluation.sandbox.access; import java.util.ArrayList; import java.util.Arrays; @@ -16,47 +16,41 @@ public final class AccessGraphBuilder { @CanIgnoreReturnValue public AccessGraphBuilder permit(AccessKey key) { - var node = createNode(key, Access.Permitted); - insert(key, node); + insert(key, Access.Permitted); return this; } @CanIgnoreReturnValue public AccessGraphBuilder deny(AccessKey key) { - var node = createNode(key, Access.Denied); - insert(key, node); + insert(key, Access.Denied); return this; } - private void insert(AccessKey key, AccessGraphNode node) { - var parent = resolveParent(key, node); - parent.insertChild(node); + private void insert(AccessKey key, Access access) { + var parent = resolveParent(key); + parent.findChildByKey(key.lastPart()) + .ifPresentOrElse( + existing -> existing.changeAccess(access), + () -> parent.insertChild(createNode(key, access)) + ); } - private AccessGraphNode resolveParent( - AccessKey key, - AccessGraphNode inserted - ) { + private AccessGraphNode resolveParent(AccessKey key) { var path = key.split(); if (path.length == 1) { return root; } var parentPath = Arrays.copyOf(path, path.length - 1); - return resolveParent(root, parentPath, 0, inserted); + return resolveParent(root, parentPath, 0); } - private AccessGraphNode resolveParent( - AccessGraphNode target, - String[] key, - int depth, - AccessGraphNode inserted - ) { + private AccessGraphNode resolveParent(AccessGraphNode target, String[] key, int depth) { if (depth >= key.length) { return target; } for (var child : target) { if (child.matchesKey(key[depth])) { - return resolveParent(child, key, depth + 1, inserted); + return resolveParent(child, key, depth + 1); } } return insertRemainingPath(target, key, depth); diff --git a/runtime/src/main/java/jsheets/sandbox/access/AccessGraphNode.java b/evaluation/src/main/java/jsheets/evaluation/sandbox/access/AccessGraphNode.java similarity index 85% rename from runtime/src/main/java/jsheets/sandbox/access/AccessGraphNode.java rename to evaluation/src/main/java/jsheets/evaluation/sandbox/access/AccessGraphNode.java index 559f991..f4eaf54 100644 --- a/runtime/src/main/java/jsheets/sandbox/access/AccessGraphNode.java +++ b/evaluation/src/main/java/jsheets/evaluation/sandbox/access/AccessGraphNode.java @@ -1,8 +1,9 @@ -package jsheets.sandbox.access; +package jsheets.evaluation.sandbox.access; import java.util.Collection; import java.util.Iterator; import java.util.Objects; +import java.util.Optional; import java.util.stream.Stream; public abstract class AccessGraphNode implements Iterable { @@ -16,9 +17,19 @@ protected AccessGraphNode(Access access, Collection nodes) { public abstract String key(); + public abstract boolean isClassMember(); public abstract boolean matchesKey(String key); public abstract boolean matchesMethod(MethodSignature signature); + public Optional findChildByKey(String key) { + for (var child : this) { + if (child.matchesKey(key)) { + return Optional.of(child); + } + } + return Optional.empty(); + } + public void changeAccess(Access access) { Objects.requireNonNull(access, "access"); this.access = access; @@ -94,6 +105,11 @@ public boolean matchesKey(String key) { public boolean matchesMethod(MethodSignature signature) { return signature.methodName().equals(key); } + + @Override + public boolean isClassMember() { + return false; + } } public static final class Method extends AccessGraphNode { @@ -126,12 +142,17 @@ public String key() { @Override public boolean matchesKey(String key) { - return false; + return method.methodName().matches(key); } @Override public boolean matchesMethod(MethodSignature signature) { - return signature.matches(method); + return method.matches(signature); + } + + @Override + public boolean isClassMember() { + return true; } } } \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/sandbox/access/AccessKey.java b/evaluation/src/main/java/jsheets/evaluation/sandbox/access/AccessKey.java similarity index 95% rename from runtime/src/main/java/jsheets/sandbox/access/AccessKey.java rename to evaluation/src/main/java/jsheets/evaluation/sandbox/access/AccessKey.java index b8a27cb..673c31f 100644 --- a/runtime/src/main/java/jsheets/sandbox/access/AccessKey.java +++ b/evaluation/src/main/java/jsheets/evaluation/sandbox/access/AccessKey.java @@ -1,4 +1,4 @@ -package jsheets.sandbox.access; +package jsheets.evaluation.sandbox.access; import java.util.Objects; import java.util.regex.Pattern; diff --git a/runtime/src/main/java/jsheets/sandbox/access/MethodSignature.java b/evaluation/src/main/java/jsheets/evaluation/sandbox/access/MethodSignature.java similarity index 52% rename from runtime/src/main/java/jsheets/sandbox/access/MethodSignature.java rename to evaluation/src/main/java/jsheets/evaluation/sandbox/access/MethodSignature.java index c77b5a1..02b282d 100644 --- a/runtime/src/main/java/jsheets/sandbox/access/MethodSignature.java +++ b/evaluation/src/main/java/jsheets/evaluation/sandbox/access/MethodSignature.java @@ -1,9 +1,10 @@ -package jsheets.sandbox.access; +package jsheets.evaluation.sandbox.access; import io.soabase.recordbuilder.core.RecordBuilder; import java.util.Collection; import java.util.List; +import java.util.Objects; import java.util.regex.Pattern; @RecordBuilder @@ -19,37 +20,29 @@ public static char methodSeparator() { return methodSeparator; } - private static final String defaultReturnType = "void"; - public static MethodSignature parse(String input) { - var trimmed = input.trim(); - int returnTypeEnd = findBeforeMethodName(trimmed, ' '); - if (returnTypeEnd < 0) { - return parseWithReturnType(input, defaultReturnType); - } - var returnType = trimmed.substring(0, returnTypeEnd); - var remaining = trimmed.substring(returnTypeEnd + 1); - return parseWithReturnType(remaining, returnType); - } - - private static int findBeforeMethodName(String input, char character) { - int firstSpace = input.indexOf(character); - int methodNameBegin = input.indexOf(methodSeparator); - return firstSpace < methodNameBegin ? firstSpace : -1; - } - - private static MethodSignature parseWithReturnType(String input, String returnType) { - var methodNameBegin = input.indexOf(methodSeparator); + var trimmed = input.trim().replace("/", "."); + var methodNameBegin = trimmed.indexOf(methodSeparator); if (methodNameBegin < 0) { throw new IllegalArgumentException( "input does not contain method name (separated by #)" ); } + return parsePreprocessed(trimmed, methodNameBegin); + } + + private static final String wildcardParameter = "*"; + + private static final Collection wildcardParameters = + List.of(wildcardParameter); + + private static MethodSignature parsePreprocessed(String input, int methodNameBegin) { var className = input.substring(0, methodNameBegin); + var returnType = parseReturnType(input); var parameterListBegin = input.indexOf('('); if (parameterListBegin < 0) { var methodName = input.substring(methodNameBegin + 1); - return new MethodSignature(className, methodName, returnType, List.of()); + return new MethodSignature(className, methodName, returnType, wildcardParameters); } var parameterListEnd = input.lastIndexOf(')'); var methodName = input.substring(methodNameBegin + 1, parameterListBegin); @@ -58,21 +51,49 @@ private static MethodSignature parseWithReturnType(String input, String returnTy return new MethodSignature(className, methodName, returnType, parameterTypes); } + private static final String wildcardReturnType = "*"; + + private static String parseReturnType(String input) { + int returnTypeBegin = input.indexOf(':'); + return returnTypeBegin < 0 + ? wildcardReturnType + : input.substring(returnTypeBegin + 1); + } + private static final Pattern parameterTypeSeparator = Pattern.compile(",\\s*"); private static Collection parseParameterTypes(String input) { return List.of(parameterTypeSeparator.split(input)); } + public MethodSignature { + Objects.requireNonNull(className, "className"); + Objects.requireNonNull(methodName, "methodName"); + Objects.requireNonNull(returnType, "returnType"); + Objects.requireNonNull(parameterTypes, "parameterTypes"); + } + public boolean matches(MethodSignature signature) { - return equals(signature); + return signature.className.equals(className) + && signature.methodName.equals(methodName) + && (hasWildcardParameters() || signature.parameterTypes.equals(parameterTypes)) + && (hasWildcardReturnType() || signature.returnType.equals(returnType)); + } + + private boolean hasWildcardReturnType() { + return wildcardReturnType.equals(returnType); + } + + public boolean hasWildcardParameters() { + return parameterTypes.size() == 1 + && parameterTypes.iterator().next().equals(wildcardParameter); } public String format() { - return "%s %s#%s".formatted( - returnType, + return "%s#%s:%s".formatted( className, - formatNameAndParameters() + formatNameAndParameters(), + returnType ); } diff --git a/evaluation/src/main/java/jsheets/evaluation/sandbox/validation/Analysis.java b/evaluation/src/main/java/jsheets/evaluation/sandbox/validation/Analysis.java new file mode 100644 index 0000000..5771aa4 --- /dev/null +++ b/evaluation/src/main/java/jsheets/evaluation/sandbox/validation/Analysis.java @@ -0,0 +1,57 @@ +package jsheets.evaluation.sandbox.validation; + +import java.util.Collection; +import java.util.Set; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public final class Analysis { + public static Analysis create() { + return new Analysis(); + } + + public interface Violation {} + + private final Collection violations = + new ConcurrentLinkedQueue<>(); + + private Analysis() {} + + public void report(Violation violation) { + violations.add(violation); + } + + public void reportViolations() { + if (!violations.isEmpty()) { + throw new FailedAnalysis(Set.copyOf(violations)); + } + } + + static final class FailedAnalysis extends RuntimeException { + private final Collection violations; + + private FailedAnalysis(Collection violations) { + super( + violations.stream() + .map(Violation::toString) + .collect(Collectors.joining(", ")) + ); + this.violations = violations; + } + + public Stream violations() { + return violations.stream(); + } + } + + public static Stream captureViolations(Throwable failure) { + while (failure != null) { + if (failure instanceof FailedAnalysis) { + return ((FailedAnalysis) failure).violations(); + } + failure = failure.getCause(); + } + return Stream.empty(); + } +} \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/sandbox/validation/ForbiddenMemberFilter.java b/evaluation/src/main/java/jsheets/evaluation/sandbox/validation/ForbiddenMemberFilter.java similarity index 54% rename from runtime/src/main/java/jsheets/sandbox/validation/ForbiddenMemberFilter.java rename to evaluation/src/main/java/jsheets/evaluation/sandbox/validation/ForbiddenMemberFilter.java index b3ec424..f8d73cf 100644 --- a/runtime/src/main/java/jsheets/sandbox/validation/ForbiddenMemberFilter.java +++ b/evaluation/src/main/java/jsheets/evaluation/sandbox/validation/ForbiddenMemberFilter.java @@ -1,11 +1,13 @@ -package jsheets.sandbox.validation; +package jsheets.evaluation.sandbox.validation; +import java.util.Arrays; import java.util.Objects; -import jsheets.sandbox.access.AccessGraph; -import jsheets.sandbox.access.AccessKey; -import jsheets.sandbox.access.MethodSignature; -import jsheets.sandbox.access.MethodSignatureBuilder; +import jsheets.evaluation.sandbox.access.AccessGraph; +import jsheets.evaluation.sandbox.access.AccessKey; +import jsheets.evaluation.sandbox.access.MethodSignature; +import jsheets.evaluation.sandbox.access.MethodSignatureBuilder; +import org.objectweb.asm.Type; public final class ForbiddenMemberFilter implements Rule { public static ForbiddenMemberFilter create(AccessGraph accessGraph) { @@ -19,7 +21,7 @@ private ForbiddenMemberFilter(AccessGraph accessGraph) { this.accessGraph = accessGraph; } - record ForbiddenMethod(MethodSignature method) implements Analysis.Violation {} + public record ForbiddenMethod(MethodSignature method) implements Analysis.Violation {} @Override public void visitCall(Analysis analysis, MethodCall call) { @@ -33,15 +35,20 @@ private MethodSignature createSignatureOfCall(MethodCall call) { return MethodSignatureBuilder.builder() .className(call.owner()) .methodName(call.method()) - .build(); + .returnType(call.type().getReturnType().getClassName()) + .parameterTypes( + Arrays.stream(call.type().getArgumentTypes()) + .map(Type::getClassName) + .toList() + ).build(); } - record ForbiddenField(String owner, String field) implements Analysis.Violation {} + public record ForbiddenField(String owner, String field) implements Analysis.Violation {} @Override public void visitFieldAccess(Analysis analysis, FieldAccess access) { - var key = "%s/%s".formatted(access.owner(), access.field()); - if (!accessGraph.isPermitted(AccessKey.slashSeparated(key))) { + var key = "%s.%s".formatted(access.owner(), access.field()); + if (!accessGraph.isPermitted(AccessKey.dotSeparated(key))) { analysis.report(new ForbiddenField(access.owner(), access.field())); } } diff --git a/evaluation/src/main/java/jsheets/evaluation/sandbox/validation/Rule.java b/evaluation/src/main/java/jsheets/evaluation/sandbox/validation/Rule.java new file mode 100644 index 0000000..f44f797 --- /dev/null +++ b/evaluation/src/main/java/jsheets/evaluation/sandbox/validation/Rule.java @@ -0,0 +1,23 @@ +package jsheets.evaluation.sandbox.validation; + +import org.objectweb.asm.Type; + +public interface Rule { + record AccessPoint(String className, String methodName) { } + + record MethodCall( + AccessPoint accessPoint, + String owner, + String method, + Type type + ) { } + + record FieldAccess( + AccessPoint accessPoint, + String owner, + String field + ) { } + + default void visitCall(Analysis analysis, MethodCall call) {} + default void visitFieldAccess(Analysis analysis, FieldAccess access) {} +} diff --git a/runtime/src/main/java/jsheets/runtime/evaluation/shell/MessageLog.java b/evaluation/src/main/java/jsheets/evaluation/shell/MessageLog.java similarity index 94% rename from runtime/src/main/java/jsheets/runtime/evaluation/shell/MessageLog.java rename to evaluation/src/main/java/jsheets/evaluation/shell/MessageLog.java index f8d3f0f..1e091e5 100644 --- a/runtime/src/main/java/jsheets/runtime/evaluation/shell/MessageLog.java +++ b/evaluation/src/main/java/jsheets/evaluation/shell/MessageLog.java @@ -1,9 +1,9 @@ -package jsheets.runtime.evaluation.shell; +package jsheets.evaluation.shell; import jsheets.EvaluateResponse; import jsheets.EvaluationResult; import jsheets.EvaluationResult.Kind; -import jsheets.runtime.evaluation.Evaluation; +import jsheets.evaluation.Evaluation; public final class MessageLog { private final Evaluation.Listener output; diff --git a/runtime/src/main/java/jsheets/runtime/evaluation/shell/MessageOutput.java b/evaluation/src/main/java/jsheets/evaluation/shell/MessageOutput.java similarity index 95% rename from runtime/src/main/java/jsheets/runtime/evaluation/shell/MessageOutput.java rename to evaluation/src/main/java/jsheets/evaluation/shell/MessageOutput.java index 708dfec..96dddb9 100644 --- a/runtime/src/main/java/jsheets/runtime/evaluation/shell/MessageOutput.java +++ b/evaluation/src/main/java/jsheets/evaluation/shell/MessageOutput.java @@ -1,4 +1,4 @@ -package jsheets.runtime.evaluation.shell; +package jsheets.evaluation.shell; import java.io.PrintStream; import java.time.Duration; @@ -9,8 +9,8 @@ import com.google.common.flogger.FluentLogger; +import jsheets.evaluation.Evaluation; import jsheets.output.CapturingOutput; -import jsheets.runtime.evaluation.Evaluation; final class MessageOutput implements AutoCloseable { private static final FluentLogger log = FluentLogger.forEnclosingClass(); diff --git a/runtime/src/main/java/jsheets/runtime/evaluation/shell/ShellEvaluation.java b/evaluation/src/main/java/jsheets/evaluation/shell/ShellEvaluation.java similarity index 96% rename from runtime/src/main/java/jsheets/runtime/evaluation/shell/ShellEvaluation.java rename to evaluation/src/main/java/jsheets/evaluation/shell/ShellEvaluation.java index 084fc8f..9382363 100644 --- a/runtime/src/main/java/jsheets/runtime/evaluation/shell/ShellEvaluation.java +++ b/evaluation/src/main/java/jsheets/evaluation/shell/ShellEvaluation.java @@ -1,4 +1,4 @@ -package jsheets.runtime.evaluation.shell; +package jsheets.evaluation.shell; import java.time.Clock; import java.util.Comparator; @@ -20,9 +20,9 @@ import jsheets.EvaluationResult; import jsheets.SnippetSources; import jsheets.StartEvaluationRequest; -import jsheets.runtime.evaluation.Evaluation; -import jsheets.shell.environment.ExecutionEnvironment; -import jsheets.shell.execution.ExecutionMethod; +import jsheets.evaluation.Evaluation; +import jsheets.evaluation.shell.environment.ExecutionEnvironment; +import jsheets.evaluation.shell.execution.ExecutionMethod; import jsheets.source.SharedSources; final class ShellEvaluation implements Evaluation { diff --git a/runtime/src/main/java/jsheets/runtime/evaluation/shell/ShellEvaluationEngine.java b/evaluation/src/main/java/jsheets/evaluation/shell/ShellEvaluationEngine.java similarity index 92% rename from runtime/src/main/java/jsheets/runtime/evaluation/shell/ShellEvaluationEngine.java rename to evaluation/src/main/java/jsheets/evaluation/shell/ShellEvaluationEngine.java index e1e96bc..a1b6eea 100644 --- a/runtime/src/main/java/jsheets/runtime/evaluation/shell/ShellEvaluationEngine.java +++ b/evaluation/src/main/java/jsheets/evaluation/shell/ShellEvaluationEngine.java @@ -1,4 +1,4 @@ -package jsheets.runtime.evaluation.shell; +package jsheets.evaluation.shell; import com.google.common.util.concurrent.ThreadFactoryBuilder; @@ -10,12 +10,12 @@ import java.util.concurrent.ScheduledExecutorService; import jsheets.StartEvaluationRequest; -import jsheets.runtime.evaluation.Evaluation; -import jsheets.runtime.evaluation.EvaluationEngine; -import jsheets.shell.environment.ExecutionEnvironment; -import jsheets.shell.environment.StandardEnvironment; -import jsheets.shell.execution.ExecutionMethod; -import jsheets.shell.execution.SystemBasedExecutionMethodFactory; +import jsheets.evaluation.Evaluation; +import jsheets.evaluation.EvaluationEngine; +import jsheets.evaluation.shell.environment.ExecutionEnvironment; +import jsheets.evaluation.shell.environment.StandardEnvironment; +import jsheets.evaluation.shell.execution.ExecutionMethod; +import jsheets.evaluation.shell.execution.SystemBasedExecutionMethodFactory; public final class ShellEvaluationEngine implements EvaluationEngine { private final Clock clock; diff --git a/runtime/src/main/java/jsheets/shell/environment/ExecutionEnvironment.java b/evaluation/src/main/java/jsheets/evaluation/shell/environment/ExecutionEnvironment.java similarity index 83% rename from runtime/src/main/java/jsheets/shell/environment/ExecutionEnvironment.java rename to evaluation/src/main/java/jsheets/evaluation/shell/environment/ExecutionEnvironment.java index 3db47f5..9619a7a 100644 --- a/runtime/src/main/java/jsheets/shell/environment/ExecutionEnvironment.java +++ b/evaluation/src/main/java/jsheets/evaluation/shell/environment/ExecutionEnvironment.java @@ -1,4 +1,4 @@ -package jsheets.shell.environment; +package jsheets.evaluation.shell.environment; import jdk.jshell.spi.ExecutionControlProvider; diff --git a/runtime/src/main/java/jsheets/shell/environment/StandardEnvironment.java b/evaluation/src/main/java/jsheets/evaluation/shell/environment/StandardEnvironment.java similarity index 96% rename from runtime/src/main/java/jsheets/shell/environment/StandardEnvironment.java rename to evaluation/src/main/java/jsheets/evaluation/shell/environment/StandardEnvironment.java index b2d1e7e..e08661e 100644 --- a/runtime/src/main/java/jsheets/shell/environment/StandardEnvironment.java +++ b/evaluation/src/main/java/jsheets/evaluation/shell/environment/StandardEnvironment.java @@ -1,4 +1,4 @@ -package jsheets.shell.environment; +package jsheets.evaluation.shell.environment; import java.net.InetAddress; import java.util.Map; diff --git a/runtime/src/main/java/jsheets/shell/environment/inprocess/EmbeddedEnvironment.java b/evaluation/src/main/java/jsheets/evaluation/shell/environment/inprocess/EmbeddedEnvironment.java similarity index 93% rename from runtime/src/main/java/jsheets/shell/environment/inprocess/EmbeddedEnvironment.java rename to evaluation/src/main/java/jsheets/evaluation/shell/environment/inprocess/EmbeddedEnvironment.java index fb126f2..2b4bb67 100644 --- a/runtime/src/main/java/jsheets/shell/environment/inprocess/EmbeddedEnvironment.java +++ b/evaluation/src/main/java/jsheets/evaluation/shell/environment/inprocess/EmbeddedEnvironment.java @@ -1,4 +1,4 @@ -package jsheets.shell.environment.inprocess; +package jsheets.evaluation.shell.environment.inprocess; import java.util.Map; @@ -6,7 +6,7 @@ import jdk.jshell.spi.ExecutionControlProvider; import jdk.jshell.spi.ExecutionEnv; import jsheets.output.TenantBasedOutput; -import jsheets.shell.environment.ExecutionEnvironment; +import jsheets.evaluation.shell.environment.ExecutionEnvironment; /** * Executing multiple shells in a shared process can have many negative diff --git a/runtime/src/main/java/jsheets/shell/environment/inprocess/InProcessExecutionControl.java b/evaluation/src/main/java/jsheets/evaluation/shell/environment/inprocess/InProcessExecutionControl.java similarity index 98% rename from runtime/src/main/java/jsheets/shell/environment/inprocess/InProcessExecutionControl.java rename to evaluation/src/main/java/jsheets/evaluation/shell/environment/inprocess/InProcessExecutionControl.java index 17074cb..439c537 100644 --- a/runtime/src/main/java/jsheets/shell/environment/inprocess/InProcessExecutionControl.java +++ b/evaluation/src/main/java/jsheets/evaluation/shell/environment/inprocess/InProcessExecutionControl.java @@ -1,4 +1,4 @@ -package jsheets.shell.environment.inprocess; +package jsheets.evaluation.shell.environment.inprocess; import java.lang.reflect.Method; import java.util.Objects; diff --git a/runtime/src/main/java/jsheets/shell/environment/inprocess/MultiTenancy.java b/evaluation/src/main/java/jsheets/evaluation/shell/environment/inprocess/MultiTenancy.java similarity index 96% rename from runtime/src/main/java/jsheets/shell/environment/inprocess/MultiTenancy.java rename to evaluation/src/main/java/jsheets/evaluation/shell/environment/inprocess/MultiTenancy.java index 50efa54..46dc26f 100644 --- a/runtime/src/main/java/jsheets/shell/environment/inprocess/MultiTenancy.java +++ b/evaluation/src/main/java/jsheets/evaluation/shell/environment/inprocess/MultiTenancy.java @@ -1,4 +1,4 @@ -package jsheets.shell.environment.inprocess; +package jsheets.evaluation.shell.environment.inprocess; import java.io.PrintStream; import java.util.function.Consumer; diff --git a/runtime/src/main/java/jsheets/shell/environment/inprocess/Preemption.java b/evaluation/src/main/java/jsheets/evaluation/shell/environment/inprocess/Preemption.java similarity index 75% rename from runtime/src/main/java/jsheets/shell/environment/inprocess/Preemption.java rename to evaluation/src/main/java/jsheets/evaluation/shell/environment/inprocess/Preemption.java index 81d5a48..1c2e7d9 100644 --- a/runtime/src/main/java/jsheets/shell/environment/inprocess/Preemption.java +++ b/evaluation/src/main/java/jsheets/evaluation/shell/environment/inprocess/Preemption.java @@ -1,4 +1,4 @@ -package jsheets.shell.environment.inprocess; +package jsheets.evaluation.shell.environment.inprocess; /** * Thrown when the execution is preempted using {@link InProcessExecutionControl#stop()}. diff --git a/runtime/src/main/java/jsheets/shell/environment/inprocess/Tenancy.java b/evaluation/src/main/java/jsheets/evaluation/shell/environment/inprocess/Tenancy.java similarity index 70% rename from runtime/src/main/java/jsheets/shell/environment/inprocess/Tenancy.java rename to evaluation/src/main/java/jsheets/evaluation/shell/environment/inprocess/Tenancy.java index bf6f910..2f3cbc6 100644 --- a/runtime/src/main/java/jsheets/shell/environment/inprocess/Tenancy.java +++ b/evaluation/src/main/java/jsheets/evaluation/shell/environment/inprocess/Tenancy.java @@ -1,4 +1,4 @@ -package jsheets.shell.environment.inprocess; +package jsheets.evaluation.shell.environment.inprocess; import jdk.jshell.spi.ExecutionEnv; diff --git a/runtime/src/main/java/jsheets/shell/execution/DirectExecution.java b/evaluation/src/main/java/jsheets/evaluation/shell/execution/DirectExecution.java similarity index 93% rename from runtime/src/main/java/jsheets/shell/execution/DirectExecution.java rename to evaluation/src/main/java/jsheets/evaluation/shell/execution/DirectExecution.java index 9186b86..15d2b67 100644 --- a/runtime/src/main/java/jsheets/shell/execution/DirectExecution.java +++ b/evaluation/src/main/java/jsheets/evaluation/shell/execution/DirectExecution.java @@ -1,4 +1,4 @@ -package jsheets.shell.execution; +package jsheets.evaluation.shell.execution; import java.util.Collection; import java.util.Objects; diff --git a/runtime/src/main/java/jsheets/shell/execution/ExecutionMethod.java b/evaluation/src/main/java/jsheets/evaluation/shell/execution/ExecutionMethod.java similarity index 84% rename from runtime/src/main/java/jsheets/shell/execution/ExecutionMethod.java rename to evaluation/src/main/java/jsheets/evaluation/shell/execution/ExecutionMethod.java index f229f45..d1bc198 100644 --- a/runtime/src/main/java/jsheets/shell/execution/ExecutionMethod.java +++ b/evaluation/src/main/java/jsheets/evaluation/shell/execution/ExecutionMethod.java @@ -1,4 +1,4 @@ -package jsheets.shell.execution; +package jsheets.evaluation.shell.execution; import java.util.Collection; diff --git a/runtime/src/main/java/jsheets/shell/execution/ExhaustiveExecution.java b/evaluation/src/main/java/jsheets/evaluation/shell/execution/ExhaustiveExecution.java similarity index 99% rename from runtime/src/main/java/jsheets/shell/execution/ExhaustiveExecution.java rename to evaluation/src/main/java/jsheets/evaluation/shell/execution/ExhaustiveExecution.java index f85d40f..9b94b65 100644 --- a/runtime/src/main/java/jsheets/shell/execution/ExhaustiveExecution.java +++ b/evaluation/src/main/java/jsheets/evaluation/shell/execution/ExhaustiveExecution.java @@ -1,4 +1,4 @@ -package jsheets.shell.execution; +package jsheets.evaluation.shell.execution; import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; diff --git a/runtime/src/main/java/jsheets/shell/execution/SystemBasedExecutionMethodFactory.java b/evaluation/src/main/java/jsheets/evaluation/shell/execution/SystemBasedExecutionMethodFactory.java similarity index 92% rename from runtime/src/main/java/jsheets/shell/execution/SystemBasedExecutionMethodFactory.java rename to evaluation/src/main/java/jsheets/evaluation/shell/execution/SystemBasedExecutionMethodFactory.java index 6fd9c3c..afe4210 100644 --- a/runtime/src/main/java/jsheets/shell/execution/SystemBasedExecutionMethodFactory.java +++ b/evaluation/src/main/java/jsheets/evaluation/shell/execution/SystemBasedExecutionMethodFactory.java @@ -1,4 +1,4 @@ -package jsheets.shell.execution; +package jsheets.evaluation.shell.execution; import jdk.jshell.JShell; diff --git a/runtime/src/main/java/jsheets/output/CapturingOutput.java b/evaluation/src/main/java/jsheets/output/CapturingOutput.java similarity index 100% rename from runtime/src/main/java/jsheets/output/CapturingOutput.java rename to evaluation/src/main/java/jsheets/output/CapturingOutput.java diff --git a/runtime/src/main/java/jsheets/output/ListeningPrintStream.java b/evaluation/src/main/java/jsheets/output/ListeningPrintStream.java similarity index 100% rename from runtime/src/main/java/jsheets/output/ListeningPrintStream.java rename to evaluation/src/main/java/jsheets/output/ListeningPrintStream.java diff --git a/runtime/src/main/java/jsheets/output/TenantBasedOutput.java b/evaluation/src/main/java/jsheets/output/TenantBasedOutput.java similarity index 100% rename from runtime/src/main/java/jsheets/output/TenantBasedOutput.java rename to evaluation/src/main/java/jsheets/output/TenantBasedOutput.java diff --git a/runtime/src/main/java/jsheets/source/SharedSources.java b/evaluation/src/main/java/jsheets/source/SharedSources.java similarity index 100% rename from runtime/src/main/java/jsheets/source/SharedSources.java rename to evaluation/src/main/java/jsheets/source/SharedSources.java diff --git a/runtime/src/main/resources/.sandbox b/evaluation/src/main/resources/.sandbox similarity index 100% rename from runtime/src/main/resources/.sandbox rename to evaluation/src/main/resources/.sandbox diff --git a/evaluation/src/test/java/jsheets/evaluation/sandbox/SandboxExecutionTest.java b/evaluation/src/test/java/jsheets/evaluation/sandbox/SandboxExecutionTest.java new file mode 100644 index 0000000..bed0abd --- /dev/null +++ b/evaluation/src/test/java/jsheets/evaluation/sandbox/SandboxExecutionTest.java @@ -0,0 +1,50 @@ +package jsheets.evaluation.sandbox; + +import jdk.jshell.JShell; +import jsheets.evaluation.sandbox.access.AccessGraph; +import jsheets.evaluation.sandbox.validation.Analysis; +import jsheets.evaluation.sandbox.validation.ForbiddenMemberFilter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +public final class SandboxExecutionTest { + @Test + public void testPermitted() { + var shell = createSandboxedShell(); + shell.eval("System.out.println(\"Hello, World!\")"); + } + + @Test + public void testDenied() { + var shell = createSandboxedShell(); + try { + shell.eval("System.err.println(\"Hello, World!\")"); + } catch (Exception failure) { + var violations = Analysis.captureViolations(failure).toList(); + Assertions.assertEquals( + List.of( + new ForbiddenMemberFilter.ForbiddenField("java.lang.System", "err") + ), + violations + ); + } + } + + private JShell createSandboxedShell() { + var accessGraph = AccessGraph.of( + "java.lang.Object", + "java.lang.System.out", + "java.io.PrintStream#println" + ); + return JShell.builder() + .out(System.out) + .err(System.err) + .executionEngine( + SandboxedEnvironment.create(List.of(ForbiddenMemberFilter.create(accessGraph))), + Map.of() + ).build(); + } +} diff --git a/runtime/src/test/java/jsheets/sandbox/AccessGraphTest.java b/evaluation/src/test/java/jsheets/evaluation/sandbox/access/AccessGraphTest.java similarity index 61% rename from runtime/src/test/java/jsheets/sandbox/AccessGraphTest.java rename to evaluation/src/test/java/jsheets/evaluation/sandbox/access/AccessGraphTest.java index a56d545..9c3a584 100644 --- a/runtime/src/test/java/jsheets/sandbox/AccessGraphTest.java +++ b/evaluation/src/test/java/jsheets/evaluation/sandbox/access/AccessGraphTest.java @@ -1,9 +1,7 @@ -package jsheets.sandbox; +package jsheets.evaluation.sandbox.access; import com.google.common.base.Charsets; -import jsheets.sandbox.access.AccessGraph; -import jsheets.sandbox.access.MethodSignature; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -11,17 +9,17 @@ import java.io.IOException; import java.util.List; -import static jsheets.sandbox.access.AccessKey.*; +import static jsheets.evaluation.sandbox.access.AccessKey.*; public class AccessGraphTest { @Test public void testReadingFromFile() { var specification = readFileContent("access-graph.txt"); var graph = AccessGraph.of(specification.split("\n")); - Assertions.assertTrue(graph.isPermitted(dotSeparated("a.b.c"))); + Assertions.assertFalse(graph.isPermitted(dotSeparated("a.b.c"))); Assertions.assertTrue(graph.isPermitted(dotSeparated("a.b.c.Foo"))); Assertions.assertTrue(graph.isPermitted(dotSeparated("a.b.c.Foo#run"))); - Assertions.assertTrue(graph.isPermitted(dotSeparated("a.b.c.Unknown"))); + Assertions.assertFalse(graph.isPermitted(dotSeparated("a.b.c.Unknown"))); Assertions.assertFalse(graph.isPermitted(dotSeparated("a.b.c.Foo#exit"))); Assertions.assertFalse(graph.isPermitted(dotSeparated("a.b.c.Bar"))); Assertions.assertFalse(graph.isPermitted(dotSeparated("a.b.c.Bar#run"))); @@ -30,10 +28,10 @@ public void testReadingFromFile() { @Test public void testOverloadedMethod() { var graph = AccessGraph.of( - "void a.b.c.Foo#run(String[], int)" + "a.b.c.Foo#run(String[], int):void" ); - Assertions.assertTrue(graph.isMethodPermitted(MethodSignature.parse("void a.b.c.Foo#run(String[], int)"))); - Assertions.assertFalse(graph.isMethodPermitted(MethodSignature.parse("void a.b.c.Foo#run"))); + Assertions.assertTrue(graph.isMethodPermitted(MethodSignature.parse("a.b.c.Foo#run(String[], int):void"))); + Assertions.assertFalse(graph.isMethodPermitted(MethodSignature.parse("a.b.c.Foo#run():void"))); } @Test @@ -65,4 +63,25 @@ private String readFileContent(String path) { throw new RuntimeException(failure); } } + + @Test + public void testOf() { + var graph = AccessGraph.of( + "java.lang", + "java.util.List", + "java.lang.Thread#sleep", + "java.lang.System", + "!java.lang.System#exit" + ); + Assertions.assertFalse(graph.isPermitted(slashSeparated("java/io"))); + Assertions.assertTrue(graph.isPermitted(slashSeparated("java/lang"))); + Assertions.assertTrue(graph.isPermitted(slashSeparated("java/util/List"))); + Assertions.assertTrue(graph.isPermitted(slashSeparated("java/lang/System/out"))); + Assertions.assertFalse( + graph.isMethodPermitted(MethodSignature.parse("java/lang/System#exit")) + ); + Assertions.assertTrue( + graph.isMethodPermitted(MethodSignature.parse("java/lang/Thread#sleep()")) + ); + } } diff --git a/runtime/src/test/java/jsheets/sandbox/MethodSignatureTest.java b/evaluation/src/test/java/jsheets/evaluation/sandbox/access/MethodSignatureTest.java similarity index 60% rename from runtime/src/test/java/jsheets/sandbox/MethodSignatureTest.java rename to evaluation/src/test/java/jsheets/evaluation/sandbox/access/MethodSignatureTest.java index b7c261d..906b006 100644 --- a/runtime/src/test/java/jsheets/sandbox/MethodSignatureTest.java +++ b/evaluation/src/test/java/jsheets/evaluation/sandbox/access/MethodSignatureTest.java @@ -1,7 +1,5 @@ -package jsheets.sandbox; +package jsheets.evaluation.sandbox.access; -import jsheets.sandbox.access.MethodSignature; -import jsheets.sandbox.access.MethodSignatureBuilder; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -12,19 +10,19 @@ public class MethodSignatureTest { public void testParsing() { Assertions.assertEquals( MethodSignatureBuilder.builder() - .className("java/lang/System") + .className("java.lang.System") .methodName("exit") .returnType("void") .parameterTypes(List.of("int")) .build(), - MethodSignature.parse("void java/lang/System#exit(int)") + MethodSignature.parse("java/lang/System#exit(int):void") ); Assertions.assertEquals( MethodSignatureBuilder.builder() - .className("java/lang/String") + .className("java.lang.String") .methodName("join") - .returnType("void") - .parameterTypes(List.of("java/lang/CharSequence", "java/lang/Iterable")) + .returnType("*") + .parameterTypes(List.of("java.lang.CharSequence", "java.lang.Iterable")) .build(), MethodSignature.parse("java/lang/String#join(java/lang/CharSequence, java/lang/Iterable)") ); diff --git a/runtime/src/test/java/jsheets/shell/ExhaustiveExecutionTest.java b/evaluation/src/test/java/jsheets/evaluation/shell/ExhaustiveExecutionTest.java similarity index 95% rename from runtime/src/test/java/jsheets/shell/ExhaustiveExecutionTest.java rename to evaluation/src/test/java/jsheets/evaluation/shell/ExhaustiveExecutionTest.java index 3c932f7..bc581b9 100644 --- a/runtime/src/test/java/jsheets/shell/ExhaustiveExecutionTest.java +++ b/evaluation/src/test/java/jsheets/evaluation/shell/ExhaustiveExecutionTest.java @@ -1,9 +1,9 @@ -package jsheets.shell; +package jsheets.evaluation.shell; import jdk.jshell.JShell; import jdk.jshell.Snippet; import jsheets.output.CapturingOutput; -import jsheets.shell.execution.ExhaustiveExecution; +import jsheets.evaluation.shell.execution.ExhaustiveExecution; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/runtime/src/test/java/jsheets/runtime/evaluation/shell/ShellEvaluationEngineTest.java b/evaluation/src/test/java/jsheets/evaluation/shell/ShellEvaluationEngineTest.java similarity index 94% rename from runtime/src/test/java/jsheets/runtime/evaluation/shell/ShellEvaluationEngineTest.java rename to evaluation/src/test/java/jsheets/evaluation/shell/ShellEvaluationEngineTest.java index 85632a4..7d55d8b 100644 --- a/runtime/src/test/java/jsheets/runtime/evaluation/shell/ShellEvaluationEngineTest.java +++ b/evaluation/src/test/java/jsheets/evaluation/shell/ShellEvaluationEngineTest.java @@ -1,4 +1,4 @@ -package jsheets.runtime.evaluation.shell; +package jsheets.evaluation.shell; import java.time.Clock; import java.util.UUID; @@ -8,8 +8,8 @@ import jsheets.Snippet; import jsheets.SnippetSources; import jsheets.StartEvaluationRequest; -import jsheets.runtime.evaluation.Evaluation; -import jsheets.shell.environment.StandardEnvironment; +import jsheets.evaluation.Evaluation; +import jsheets.evaluation.shell.environment.StandardEnvironment; import org.junit.jupiter.api.Test; /* This is not a Unit Test */ diff --git a/runtime/src/test/java/jsheets/output/TenantBasedOutputTest.java b/evaluation/src/test/java/jsheets/output/TenantBasedOutputTest.java similarity index 100% rename from runtime/src/test/java/jsheets/output/TenantBasedOutputTest.java rename to evaluation/src/test/java/jsheets/output/TenantBasedOutputTest.java diff --git a/runtime/src/test/resources/access-graph.txt b/evaluation/src/test/resources/access-graph.txt similarity index 100% rename from runtime/src/test/resources/access-graph.txt rename to evaluation/src/test/resources/access-graph.txt diff --git a/protocol/build.gradle b/protocol/build.gradle index a3d5377..53600e7 100644 --- a/protocol/build.gradle +++ b/protocol/build.gradle @@ -12,6 +12,8 @@ repositories { } dependencies { + compileOnly "io.grpc:grpc-all:$grpcVersion" + compileOnly "javax.annotation:javax.annotation-api:$javaxAnnotationVersion" implementation "com.google.protobuf:protobuf-java:$protobufJavaVersion" } @@ -58,6 +60,7 @@ protobuf { outputSubDir = 'js' } } + "grpc" {} } all()*.builtins { "js" { diff --git a/runtime/build.gradle b/runtime/build.gradle index 0f5be1e..758dec9 100644 --- a/runtime/build.gradle +++ b/runtime/build.gradle @@ -1,35 +1,95 @@ plugins { id 'java' + id 'application' } group 'dev.jsheets' version '0.1.0' -sourceCompatibility = 16 -targetCompatibility = 16 - repositories { mavenCentral() } +sourceCompatibility = 16 +targetCompatibility = 16 + ext { - asmVersion = '9.2' - recordBuilderVersion = '26' + mongoDbDriverVersion = '4.3.2' + cfg4jVersion = '4.4.1' + slf4jVersion = '1.7.32' + javalinVersion = '4.1.1' + argsParseVersion = '0.9.0' + guiceVersion = '5.0.1' } dependencies { implementation project(':protocol') - implementation "org.ow2.asm:asm:$asmVersion" + implementation project(':evaluation') + implementation "com.ecwid.consul:consul-api:$consuleClientVersion" + implementation "io.micrometer:micrometer-core:$micrometerVersion" + implementation "org.mongodb:mongodb-driver-sync:$mongoDbDriverVersion" + implementation "org.cfg4j:cfg4j-core:$cfg4jVersion" + implementation "org.slf4j:slf4j-simple:$slf4jVersion" + implementation "io.grpc:grpc-all:$grpcVersion" implementation "com.google.protobuf:protobuf-java-util:$protobufJavaVersion" + implementation "net.sourceforge.argparse4j:argparse4j:$argsParseVersion" + implementation "javax.annotation:javax.annotation-api:$javaxAnnotationVersion" implementation "com.google.flogger:flogger:$floggerVersion" - testImplementation "org.junit.jupiter:junit-jupiter-api:$junitPlatformVersion" - testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:$junitPlatformVersion" - testRuntimeOnly "com.google.flogger:flogger-slf4j-backend:$floggerVersion" + implementation "com.google.flogger:flogger-slf4j-backend:$floggerVersion" + implementation "com.google.inject:guice:$guiceVersion" compileOnly "io.soabase.record-builder:record-builder-core:$recordBuilderVersion" annotationProcessor "io.soabase.record-builder:record-builder-processor:$recordBuilderVersion" + testImplementation "org.junit.jupiter:junit-jupiter-api:$junitPlatformVersion" + testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:$junitPlatformVersion" } test { useJUnitPlatform() - jvmArgs += ['--add-opens', 'jdk.jshell/jdk.jshell=ALL-UNNAMED'] -} \ No newline at end of file +} + +mainClassName = 'jsheets.runtime.App' + +task copyDependencies(type: Copy) { + from configurations.runtimeClasspath + into "$buildDir/libs" +} + +private def listClassPath() { + return configurations.runtimeClasspath.collect { "libs/${it.name}" }.join(" ") +} + +processResources { + duplicatesStrategy = DuplicatesStrategy.INCLUDE +} + +jar { + duplicatesStrategy = DuplicatesStrategy.INCLUDE + manifest { + attributes( + "Main-Class": "jsheets.runtime.App", + "Class-Path": listClassPath() + ) + } +} + +def dockerImageName = "ehenoma/jsheets-runtime" +def dockerImageTag = System.getenv("TARGET_IMAGE_TAG") || "latest" +def dockerImage = "$dockerImageName:$dockerImageTag" + +task buildDocker(type: Exec) { + dependsOn copyDependencies, build + workingDir "$projectDir" + commandLine "docker", "build", "--rm", ".", "-t", dockerImage, "-f", "./server/deploy/Dockerfile" +} + +task justRunDocker(type: Exec) { + workingDir "$projectDir" + commandLine "docker", "run", dockerImage, "-p", "8080:8080" +} + +task runDocker(type: Exec) { + dependsOn buildDocker + workingDir "$projectDir" + commandLine "docker", "run", dockerImage, "-p", "8080:8080" +} + diff --git a/runtime/deploy/Dockerfile b/runtime/deploy/Dockerfile new file mode 100644 index 0000000..aff840f --- /dev/null +++ b/runtime/deploy/Dockerfile @@ -0,0 +1,47 @@ +FROM openjdk:17-alpine AS link-java + +MAINTAINER "merlinosayimwen@gmail.com" + +ARG APP_HEAP_LIMIT=1G +ENV APP_HEAP_LIMIT=$APP_HEAP_LIMIT + +ARG APP_HEAP_MINIMUM=512m +ENV APP_HEAP_MINIMUM=$APP_HEAP_MINIMUM + +ARG APP_LOG_VERSION=false +ENV APP_LOG_VERSION=$APP_LOG_VERSION + +WORKDIR /usr/src/build + +# Context is parent directory +COPY ./runtime/build/libs libs + +RUN mv ./libs/app.jar app.jar + +RUN jdeps --ignore-missing-deps -q --multi-release 17 \ + --print-module-deps \ + --class-path libs/* \ + app.jar > deps.info + +# link using zip compression +RUN jlink --verbose \ + --compress 2 \ + --strip-java-debug-attributes \ + --no-header-files \ + --no-man-pages \ + --output jre \ + --add-modules $(cat deps.info) + +FROM alpine:latest +WORKDIR /app + +COPY --from=link-java /usr/src/build/jre jre +COPY --from=link-java /usr/src/build/libs/* libs/ +COPY --from=link-java /usr/src/build/app.jar app.jar +ADD /server/deploy/entrypoint.sh entrypoint.sh +ADD /website/build static/ +RUN chmod +x entrypoint.sh + +EXPOSE 8080 + +ENTRYPOINT ./entrypoint.sh \ No newline at end of file diff --git a/runtime/deploy/entrypoint.sh b/runtime/deploy/entrypoint.sh new file mode 100644 index 0000000..957d043 --- /dev/null +++ b/runtime/deploy/entrypoint.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env sh + +if [ "$APP_LOG_VERSION" = "true" ]; +then + java -version +fi + +# jdk.jshell is opened to support the exhaustive-execution + +jre/bin/java \ + --add-opens jdk.jshell/jdk.jshell=ALL-UNNAMED \ + -jar app.jar \ + -XX:+UseZGC \ + -XX:+UseZGC \ + -Xmx"$APP_HEAP_LIMIT" \ + -Xms"$APP_HEAP_MINIMUM" \ + "$@" \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/config/CamelCase.java b/runtime/src/main/java/jsheets/config/CamelCase.java new file mode 100644 index 0000000..123913f --- /dev/null +++ b/runtime/src/main/java/jsheets/config/CamelCase.java @@ -0,0 +1,55 @@ +package jsheets.config; + +public final class CamelCase { + private CamelCase() {} + + @FunctionalInterface + public interface Case { + static Case upper() { + return Character::toUpperCase; + } + + static Case lower() { + return Character::toLowerCase; + } + + static Case same() { + return character -> character; + } + + char apply(char character); + } + + public static String convert( + String input, + String separator, + Case characterCase + ) { + return switch (input.length()) { + case 0 -> input; + case 1 -> String.valueOf(characterCase.apply(input.charAt(0))); + default -> convertWithSufficientLength(input, separator, characterCase); + }; + } + + private static String convertWithSufficientLength( + String input, + String separator, + Case characterCase + ) { + var output = new StringBuilder(input.length()); + char firstCharacter = input.charAt(0); + boolean lastCapitalized = Character.isUpperCase(firstCharacter); + output.append(characterCase.apply(firstCharacter)); + for (int index = 1; index < input.length(); index++) { + char character = input.charAt(index); + boolean capitalized = Character.isUpperCase(character); + if (capitalized && !lastCapitalized) { + output.append(separator); + } + lastCapitalized = capitalized; + output.append(characterCase.apply(character)); + } + return output.toString(); + } +} \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/config/CombinedConfig.java b/runtime/src/main/java/jsheets/config/CombinedConfig.java new file mode 100644 index 0000000..df49e4c --- /dev/null +++ b/runtime/src/main/java/jsheets/config/CombinedConfig.java @@ -0,0 +1,43 @@ +package jsheets.config; + +import java.util.Arrays; +import java.util.Objects; + +public final class CombinedConfig implements Config { + /** + * Combines the {@code configs}, prioritized based on descending order. + *

+ * If config {@code Ci} contains a value for any given key, then it is + * returned, otherwise config {@code Ci+1} is queried until no configs remain + * and a missing field is returned. + * + * @param configs Ordered set of sources. + * @return Config that merges the values of all {@code configs}. + */ + public static Config of(Config... configs) { + Objects.requireNonNull(configs); + return new CombinedConfig(configs.clone()); + } + + private final Config[] configs; + + private CombinedConfig(Config[] configs) { + this.configs = configs; + } + + @Override + public Field lookup(Key key) { + for (var config : configs) { + var field = config.lookup(key); + if (field.exists()) { + return field; + } + } + return new MissingField<>(key); + } + + @Override + public String toString() { + return "CombinedSource(%s)".formatted(Arrays.toString(configs)); + } +} \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/config/Config.java b/runtime/src/main/java/jsheets/config/Config.java new file mode 100644 index 0000000..d64c2df --- /dev/null +++ b/runtime/src/main/java/jsheets/config/Config.java @@ -0,0 +1,93 @@ +package jsheets.config; + +import java.util.Objects; +import java.util.Optional; +import java.util.function.Function; + +/** + * Storage of {@link Config.Field configuration fields}, that can be accessed + * in a typesafe way (using the {@link Config.Key keys}. + */ +public interface Config { + Field lookup(Key key); + + interface Source { + Config load(); + } + + interface Field { + T require(); + + Optional orNone(); + + default T or(T value) { + return orNone().orElse(value); + } + + boolean exists(); + } + + final class Key { + public static Key ofFlag(String name) { + Objects.requireNonNull(name, "name"); + return new Key<>(name, Boolean::parseBoolean); + } + + public static Key ofString(String name) { + Objects.requireNonNull(name, "name"); + return new Key<>(name, String::valueOf); + } + + public static Key ofInt(String name) { + Objects.requireNonNull(name, "name"); + return new Key<>(name, Integer::parseInt); + } + + public static Key ofDouble(String name) { + Objects.requireNonNull(name, "name"); + return new Key<>(name, Double::parseDouble); + } + + public static Key of(String name, Function parse) { + Objects.requireNonNull(name, "name"); + Objects.requireNonNull(parse, "parse"); + return new Key<>(name, parse); + } + + private final String name; + private final Function parse; + + private Key(String name, Function parse) { + this.name = name; + this.parse = parse; + } + + public Field in(Config config) { + return config.lookup(this); + } + + public T parse(String rawInput) { + return parse.apply(rawInput); + } + + @Override + public boolean equals(Object target) { + if (target == this) { + return true; + } + return target instanceof Key key && ( + key.name.equals(name) + ); + } + + @Override + public int hashCode() { + return Objects.hash(name); + } + + @Override + public String toString() { + return name; + } + } +} diff --git a/runtime/src/main/java/jsheets/config/EnvironmentConfig.java b/runtime/src/main/java/jsheets/config/EnvironmentConfig.java new file mode 100644 index 0000000..c68e707 --- /dev/null +++ b/runtime/src/main/java/jsheets/config/EnvironmentConfig.java @@ -0,0 +1,44 @@ +package jsheets.config; + +import java.util.Map; +import java.util.Objects; + +import com.google.common.annotations.VisibleForTesting; + +public final class EnvironmentConfig implements Config { + public static Config.Source prefixed(String prefix) { + Objects.requireNonNull(prefix, "prefix"); + return () -> new EnvironmentConfig(prefix + "_", System.getenv()); + } + + private final String prefix; + private final Map environment; + + @VisibleForTesting + EnvironmentConfig(String prefix, Map environment) { + this.prefix = prefix; + this.environment = environment; + } + + @Override + public Field lookup(Key key) { + var value = environment.get(formatKey(key)); + if (value == null) { + return new MissingField<>(key); + } + return new ResolvedField<>(key.parse(value)); + } + + private String formatKey(Key key) { + return prefix + translateKey(key.toString()); + } + + @VisibleForTesting + static String translateKey(String key) { + return toEnvironmentCase(key).replace(".", "_"); + } + + private static String toEnvironmentCase(String input) { + return CamelCase.convert(input, "_", Character::toUpperCase); + } +} \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/config/MissingField.java b/runtime/src/main/java/jsheets/config/MissingField.java new file mode 100644 index 0000000..f9039fa --- /dev/null +++ b/runtime/src/main/java/jsheets/config/MissingField.java @@ -0,0 +1,26 @@ +package jsheets.config; + +import java.util.NoSuchElementException; +import java.util.Optional; + +record MissingField(Config.Key key) implements Config.Field { + @Override + public T require() { + throw new NoSuchElementException(key + " is missing"); + } + + @Override + public T or(T value) { + return value; + } + + @Override + public Optional orNone() { + return Optional.empty(); + } + + @Override + public boolean exists() { + return false; + } +} \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/config/RawConfig.java b/runtime/src/main/java/jsheets/config/RawConfig.java new file mode 100644 index 0000000..f307ec5 --- /dev/null +++ b/runtime/src/main/java/jsheets/config/RawConfig.java @@ -0,0 +1,63 @@ +package jsheets.config; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +import com.google.errorprone.annotations.CanIgnoreReturnValue; + +public final class RawConfig implements Config { + public static RawConfig of(Map entries) { + var fields = new HashMap(entries.size()); + for (var entry : entries.entrySet()) { + Objects.requireNonNull(entry.getKey(), "keys may not be null"); + Objects.requireNonNull(entry.getValue(), "values may not be null"); + fields.put(entry.getKey(), entry.getValue()); + } + return new RawConfig(fields); + } + + private final Map values; + + private RawConfig(Map values) { + this.values = values; + } + + @Override + public Field lookup(Key key) { + Objects.requireNonNull(key, "key may not be null"); + var value = values.get(key.toString()); + if (value == null) { + return new MissingField<>(key); + } + return new ResolvedField<>(key.parse(value)); + } + + public static Builder newBuilder() { + return new Builder(); + } + + public static final class Builder { + private final Map values = new HashMap<>(); + + @CanIgnoreReturnValue + public Builder with(Key key, T value) { + Objects.requireNonNull(key, "keys may not be null"); + Objects.requireNonNull(value, "values may not be null"); + values.put(key.toString(), value.toString()); + return this; + } + + @CanIgnoreReturnValue + public Builder withRaw(String key, String value) { + Objects.requireNonNull(key, "keys may not be null"); + Objects.requireNonNull(value, "values may not be null"); + values.put(key, value); + return this; + } + + public RawConfig create() { + return new RawConfig(Map.copyOf(values)); + } + } +} \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/config/ResolvedField.java b/runtime/src/main/java/jsheets/config/ResolvedField.java new file mode 100644 index 0000000..ab7fd89 --- /dev/null +++ b/runtime/src/main/java/jsheets/config/ResolvedField.java @@ -0,0 +1,25 @@ +package jsheets.config; + +import java.util.Optional; + +record ResolvedField(T value) implements Config.Field { + @Override + public T require() { + return value; + } + + @Override + public T or(T fallback) { + return value; + } + + @Override + public Optional orNone() { + return Optional.ofNullable(value); + } + + @Override + public boolean exists() { + return true; + } +} \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/config/consul/ConsulConfigSource.java b/runtime/src/main/java/jsheets/config/consul/ConsulConfigSource.java new file mode 100644 index 0000000..f19dbcb --- /dev/null +++ b/runtime/src/main/java/jsheets/config/consul/ConsulConfigSource.java @@ -0,0 +1,50 @@ +package jsheets.config.consul; + +import java.util.Objects; + +import com.ecwid.consul.v1.ConsulClient; + +import jsheets.config.Config; +import jsheets.config.RawConfig; + +/** + * Loads configuration from a Consul backend. + */ +public final class ConsulConfigSource implements Config.Source { + public static ConsulConfigSource prefixed(String prefix, ConsulClient client) { + Objects.requireNonNull(client, "client"); + Objects.requireNonNull(prefix, "prefix"); + return new ConsulConfigSource(prefix + ".", client); + } + + private final String prefix; + private final ConsulClient client; + + private ConsulConfigSource(String prefix, ConsulClient client) { + this.prefix = prefix; + this.client = client; + } + + @Override + public Config load() { + var pairs = client.getKVValues(prefix).getValue(); + var config = RawConfig.newBuilder(); + for (var pair : pairs) { + var key = removePrefix(pair.getKey()); + config.withRaw(key, pair.getDecodedValue()); + } + return config.create(); + } + + private String removePrefix(String key) { + return key.startsWith(prefix) + ? key.substring(prefix.length()) + : key; + } + + @Override + public String toString() { + return "ConsulConfigSource(prefix=%s, client=%s)" + .formatted(prefix, client); + } +} \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/runtime/App.java b/runtime/src/main/java/jsheets/runtime/App.java new file mode 100644 index 0000000..d9a6a60 --- /dev/null +++ b/runtime/src/main/java/jsheets/runtime/App.java @@ -0,0 +1,53 @@ +package jsheets.runtime; + +import java.io.IOException; + +import com.google.common.flogger.FluentLogger; + +import com.google.inject.Guice; +import com.google.inject.Injector; + +import jsheets.runtime.evaluation.EvaluationModule; + +public final class App { + private static final FluentLogger log = FluentLogger.forEnclosingClass(); + + private App() {} + + public static void main(String[] options) { + configureLogging(); + var injector = configureInjector(); + launch(injector); + } + + private static void launch(Injector injector) { + var setup = injector.getInstance(ServerSetup.class); + try { + setup.start(); + } catch (IOException failure) { + log.atSevere() + .withCause(failure) + .log("encountered error while serving"); + } catch (InterruptedException interruption) { + log.atSevere() + .withCause(interruption) + .log("interrupted while serving"); + } + } + + private static Injector configureInjector() { + return Guice.createInjector( + ServerSetupModule.create(), + ConfigModule.create(), + ConsulModule.create(), + EvaluationModule.create() + ); + } + + private static void configureLogging() { + System.setProperty( + "flogger.backend_factory", + "com.google.common.flogger.backend.slf4j.Slf4jBackendFactory#getInstance" + ); + } +} \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/runtime/ConfigModule.java b/runtime/src/main/java/jsheets/runtime/ConfigModule.java new file mode 100644 index 0000000..b117569 --- /dev/null +++ b/runtime/src/main/java/jsheets/runtime/ConfigModule.java @@ -0,0 +1,45 @@ +package jsheets.runtime; + +import java.util.ArrayList; +import java.util.Optional; + +import com.google.inject.AbstractModule; +import com.google.inject.Provides; +import com.google.inject.Singleton; + +import javax.inject.Named; +import jsheets.config.CombinedConfig; +import jsheets.config.Config; +import jsheets.config.EnvironmentConfig; +import jsheets.config.consul.ConsulConfigSource; +import jsheets.runtime.evaluation.SandboxConfigSource; + +final class ConfigModule extends AbstractModule { + static ConfigModule create() { + return new ConfigModule(); + } + + private ConfigModule() {} + + private static final String environmentPrefix = "JSHELL_RUNTIME"; + + @Provides + @Singleton + Config createConfig( + Optional consulSource, + @Named("environment") Config environment + ) { + var configs = new ArrayList(); + configs.add(environment); + configs.add(SandboxConfigSource.fromClassPath().load()); + consulSource.ifPresent(source -> configs.add(source.load())); + return CombinedConfig.of(configs.toArray(Config[]::new)); + } + + @Provides + @Singleton + @Named("environment") + Config environmentConfig() { + return EnvironmentConfig.prefixed(environmentPrefix).load(); + } +} \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/runtime/ConsulModule.java b/runtime/src/main/java/jsheets/runtime/ConsulModule.java new file mode 100644 index 0000000..804ff63 --- /dev/null +++ b/runtime/src/main/java/jsheets/runtime/ConsulModule.java @@ -0,0 +1,58 @@ +package jsheets.runtime; + +import java.util.Optional; + +import com.google.common.net.HostAndPort; + +import com.google.inject.AbstractModule; +import com.google.inject.Provides; +import com.google.inject.Singleton; + +import com.ecwid.consul.v1.ConsulClient; +import javax.inject.Named; +import jsheets.config.Config; +import jsheets.config.consul.ConsulConfigSource; +import jsheets.runtime.discovery.ConsulServiceAdvertisementChannel; +import jsheets.runtime.discovery.ServiceAdvertisementChannel; + +final class ConsulModule extends AbstractModule { + static ConsulModule create() { + return new ConsulModule(); + } + + private ConsulModule() {} + + private static final Config.Key consulEndpointKey = + Config.Key.of("consul.endpoint", HostAndPort::fromString); + + @Provides + @Singleton + Optional consulClient(@Named("environment") Config config) { + return consulEndpointKey.in(config).orNone().map(endpoint -> + new ConsulClient(endpoint.getHost(), endpoint.getPort()) + ); + } + + private static final String consulKeyPrefix = "jsheets.runtime"; + + @Provides + @Singleton + Optional consulConfigSource( + Optional clientBinding + ) { + return clientBinding.map(client -> + ConsulConfigSource.prefixed(consulKeyPrefix, client) + ); + } + + @Provides + @Singleton + Optional consulAdvertisementChannel( + @Named("serviceId") String serviceId, + Optional clientBinding + ) { + return clientBinding.map(client -> + ConsulServiceAdvertisementChannel.forServiceId(serviceId, client) + ); + } +} \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/runtime/ServerSetup.java b/runtime/src/main/java/jsheets/runtime/ServerSetup.java new file mode 100644 index 0000000..242929a --- /dev/null +++ b/runtime/src/main/java/jsheets/runtime/ServerSetup.java @@ -0,0 +1,129 @@ +package jsheets.runtime; + +import java.io.IOException; +import java.util.Collection; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; + +import com.google.common.flogger.FluentLogger; +import com.google.common.flogger.MetadataKey; + +import io.grpc.Server; +import io.grpc.ServerBuilder; +import io.grpc.ServerServiceDefinition; +import io.grpc.health.v1.HealthCheckResponse; +import io.grpc.protobuf.services.HealthStatusManager; +import io.grpc.protobuf.services.ProtoReflectionService; +import io.soabase.recordbuilder.core.RecordBuilder; +import javax.inject.Inject; +import javax.inject.Named; + +public final class ServerSetup { + private static final FluentLogger log = FluentLogger.forEnclosingClass(); + + public enum Feature { + Reflection, + Health + } + + @RecordBuilder + public record Options(int port, Set features) {} + + public interface Hook { + void start(); + void stop(); + } + + private final Options options; + private final ServerServiceDefinition runtimeService; + private final HealthStatusManager health = new HealthStatusManager(); + private final AtomicReference activeServer = new AtomicReference<>(null); + private final Collection hooks; + + @Inject + ServerSetup( + Options options, + Collection hooks, + @Named("runtimeService") ServerServiceDefinition runtimeService + ) { + this.hooks = hooks; + this.options = options; + this.runtimeService = runtimeService; + } + + public void start() throws IOException, InterruptedException { + log.atConfig().log("configuring server"); + var server = createServer(); + if (!activeServer.compareAndSet(null, server)) { + throw new IllegalStateException("already running"); + } + boot(server); + try { + server.awaitTermination(); + } finally { + callStopHooks(); + } + } + + private static final MetadataKey portKey = + MetadataKey.single("port", Integer.class); + + private void boot(Server server) throws IOException { + server.start(); + log.atInfo().with(portKey, options.port).log("listening for requests"); + callStartHooks(); + updateHealth(HealthCheckResponse.ServingStatus.SERVING); + log.atConfig().log("finished boot"); + } + + private void callStartHooks() { + for (var hook : hooks) { + try { + log.atConfig().log("calling start() in hook %s", hook); + hook.start(); + } catch (Throwable failure) { + log.atWarning().withCause(failure).log( + "error while calling start() in hook %s", + hook + ); + } + } + } + + private void callStopHooks() { + for (var hook : hooks) { + try { + log.atConfig().log("calling stop() in hook %s", hook); + hook.stop(); + } catch (Throwable failure) { + log.atWarning().withCause(failure).log( + "error while calling stop() in hook %s", + hook + ); + } + } + } + + private void updateHealth(HealthCheckResponse.ServingStatus status) { + health.setStatus(runtimeService.getServiceDescriptor().getName(), status); + } + + private Server createServer() { + var server = ServerBuilder.forPort(options.port); + server.addService(runtimeService); + addOptionalServices(server); + return server.build(); + } + + private void addOptionalServices(ServerBuilder server) { + if (options.features().contains(Feature.Reflection)) { + log.atConfig().log("the grpc reflection-service has been enabled"); + server.addService(ProtoReflectionService.newInstance()); + } + if (options.features().contains(Feature.Health)) { + log.atConfig().log("the grpc health-service has been enabled"); + updateHealth(HealthCheckResponse.ServingStatus.NOT_SERVING); + server.addService(health.getHealthService()); + } + } +} \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/runtime/ServerSetupModule.java b/runtime/src/main/java/jsheets/runtime/ServerSetupModule.java new file mode 100644 index 0000000..c6db93e --- /dev/null +++ b/runtime/src/main/java/jsheets/runtime/ServerSetupModule.java @@ -0,0 +1,96 @@ +package jsheets.runtime; + +import java.util.Collection; +import java.util.EnumSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.function.Supplier; + +import com.google.common.net.HostAndPort; + +import com.google.inject.AbstractModule; +import com.google.inject.Provides; + +import io.grpc.ServerServiceDefinition; +import javax.inject.Named; +import javax.inject.Singleton; +import jsheets.config.Config; +import jsheets.runtime.discovery.AdvertisementHook; +import jsheets.runtime.discovery.ServiceAdvertisementChannel; + +/* Configures the ServerSetup. */ +final class ServerSetupModule extends AbstractModule { + static ServerSetupModule create() { + return new ServerSetupModule(); + } + + private ServerSetupModule() {} + + @Provides + @Singleton + @Named("runtimeService") + ServerServiceDefinition snippetRuntimeService(SnippetRuntimeService service) { + return service.bindService(); + } + + private static final Config.Key servicePortKey = + Config.Key.ofInt("server.port"); + + private static final int defaultPort = 8080; + + @Provides + @Singleton + ServerSetup.Options serverSetupOptions(Config config) { + return ServerSetupOptionsBuilder.builder() + .port(servicePortKey.in(config).orNone().orElse(defaultPort)) + .features(listFeatures(config)) + .build(); + } + + private static final Config.Key healthCheckKey = + Config.Key.ofFlag("server.features.enableHealthCheck"); + + private static final Config.Key grpcReflectionKey = + Config.Key.ofFlag("server.features.enableGrpcReflection"); + + private Set listFeatures(Config config) { + var features = EnumSet.noneOf(ServerSetup.Feature.class); + if (healthCheckKey.in(config).orNone().orElse(true)) { + features.add(ServerSetup.Feature.Health); + } + if (grpcReflectionKey.in(config).orNone().orElse(false)) { + features.add(ServerSetup.Feature.Reflection); + } + return features; + } + + private static final Config.Key serviceIdKey = + Config.Key.ofString("service.id"); + + @Provides + @Singleton + @Named("serviceId") + String serviceId(Config config) { + return serviceIdKey.in(config).orNone() + .orElse(UUID.randomUUID().toString()); + } + + private static final Config.Key advertisedHostKey = + Config.Key.of("service.advertisedHost", HostAndPort::fromString); + + @Provides + @Singleton + Collection setupHooks( + Config config, + Supplier advertisementChannelFactory + ) { + return advertisedHostKey.in(config).orNone().map(host -> { + var hook = AdvertisementHook.create( + host, + advertisementChannelFactory.get() + ); + return List.of(hook); + }).orElse(List.of()); + } +} \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/runtime/SnippetRuntimeService.java b/runtime/src/main/java/jsheets/runtime/SnippetRuntimeService.java new file mode 100644 index 0000000..d60b3db --- /dev/null +++ b/runtime/src/main/java/jsheets/runtime/SnippetRuntimeService.java @@ -0,0 +1,109 @@ +package jsheets.runtime; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import com.google.common.flogger.FluentLogger; + +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import javax.inject.Inject; +import jsheets.EvaluateRequest; +import jsheets.EvaluateResponse; +import jsheets.SnippetRuntimeGrpc.SnippetRuntimeImplBase; +import jsheets.StartEvaluationRequest; +import jsheets.StopEvaluationRequest; +import jsheets.evaluation.Evaluation; +import jsheets.evaluation.EvaluationEngine; + +final class SnippetRuntimeService extends SnippetRuntimeImplBase { + private static final FluentLogger log = FluentLogger.forEnclosingClass(); + + private final EvaluationEngine engine; + + @Inject + SnippetRuntimeService(EvaluationEngine engine) { + this.engine = engine; + } + + @Override + public StreamObserver evaluate( + StreamObserver responseStream + ) { + return new Call(responseStream); + } + + final class Call + implements StreamObserver, Evaluation.Listener { + + private final StreamObserver responseStream; + private final Lock lock = new ReentrantLock(); + private Evaluation evaluation; + + private Call(StreamObserver responseStream) { + this.responseStream = responseStream; + } + + @Override + public void onNext(EvaluateRequest request) { + lock.lock(); + try { + switch (request.getMessageCase()) { + case STOP -> processStop(request.getStop()); + case START -> processStart(request.getStart()); + default -> processUnknown(request); + } + } finally { + lock.unlock(); + } + } + + private static final Status invalidState = + Status.FAILED_PRECONDITION.withDescription("state"); + + private void processStop(StopEvaluationRequest request) { + if (evaluation == null) { + log.atWarning().log("received stop request without active evaluation"); + responseStream.onError(invalidState.asException()); + return; + } + evaluation.stop(); + log.atFine().log("closed evaluation"); + } + + private void processStart(StartEvaluationRequest request) { + if (evaluation != null) { + log.atWarning().log("received start request with active evaluation"); + responseStream.onError(invalidState.asException()); + return; + } + evaluation = engine.start(request, this); + log.atFine().log("started evaluation"); + } + + private void processUnknown(EvaluateRequest request) { + log.atWarning() + .atMostEvery(5, TimeUnit.SECONDS) + .log("received unknown request message: %s", request); + } + + @Override + public void onError(Throwable failure) {} + + @Override + public void onCompleted() {} + + // Listens to EvaluationEngine + @Override + public void send(EvaluateResponse response) { + responseStream.onNext(response); + } + + // Listens to EvaluationEngine + @Override + public void close() { + responseStream.onCompleted(); + } + } +} \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/runtime/discovery/AdvertisementHook.java b/runtime/src/main/java/jsheets/runtime/discovery/AdvertisementHook.java new file mode 100644 index 0000000..6c54647 --- /dev/null +++ b/runtime/src/main/java/jsheets/runtime/discovery/AdvertisementHook.java @@ -0,0 +1,51 @@ +package jsheets.runtime.discovery; + +import java.util.Objects; + +import com.google.common.net.HostAndPort; + +import javax.inject.Inject; +import jsheets.runtime.ServerSetup; + +public final class AdvertisementHook implements ServerSetup.Hook { + public static AdvertisementHook create( + HostAndPort advertisedHost, + ServiceAdvertisementChannel channel + ) { + Objects.requireNonNull(advertisedHost, "advertisedHost"); + Objects.requireNonNull(channel, "channel"); + return new AdvertisementHook(advertisedHost, channel); + } + + private volatile ServiceAdvertisement advertisement; + private final HostAndPort advertisedHost; + private final ServiceAdvertisementChannel advertisementChannel; + + @Inject + AdvertisementHook( + HostAndPort advertisedHost, + ServiceAdvertisementChannel advertisementChannel + ) { + this.advertisedHost = advertisedHost; + this.advertisementChannel = advertisementChannel; + } + + @Override + public void start() { + advertisement = advertisementChannel.advertise(advertisedHost); + } + + @Override + public void stop() { + var currentAdvertisement = advertisement; + if (currentAdvertisement != null) { + currentAdvertisement.remove(); + advertisement = null; + } + } + + @Override + public String toString() { + return "AdvertisementHook(host=%s)".formatted(advertisedHost); + } +} \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/runtime/discovery/ConsulServiceAdvertisementChannel.java b/runtime/src/main/java/jsheets/runtime/discovery/ConsulServiceAdvertisementChannel.java new file mode 100644 index 0000000..ec257e4 --- /dev/null +++ b/runtime/src/main/java/jsheets/runtime/discovery/ConsulServiceAdvertisementChannel.java @@ -0,0 +1,82 @@ +package jsheets.runtime.discovery; + +import java.util.Objects; + +import com.google.common.flogger.FluentLogger; +import com.google.common.flogger.MetadataKey; +import com.google.common.net.HostAndPort; + +import com.ecwid.consul.v1.ConsulClient; +import com.ecwid.consul.v1.agent.model.NewService; + +/** + * Implements a Consul + * backend for Service Discovery. + *

+ * Consul discovery uses additional gRpc health checks + * that can only be used if the {@code Health} Feature is activated. + */ +public final class ConsulServiceAdvertisementChannel implements ServiceAdvertisementChannel { + private static final FluentLogger log = FluentLogger.forEnclosingClass(); + + public static ConsulServiceAdvertisementChannel forServiceId( + String serviceId, + ConsulClient client + ) { + Objects.requireNonNull(serviceId, "serviceId"); + Objects.requireNonNull(client, "client"); + return new ConsulServiceAdvertisementChannel(serviceId, client); + } + + private final String serviceId; + private final ConsulClient client; + + private ConsulServiceAdvertisementChannel(String serviceId, ConsulClient client) { + this.client = client; + this.serviceId = serviceId; + } + + private static final MetadataKey idKey = + MetadataKey.single("serviceId", String.class); + + @Override + public ServiceAdvertisement advertise(HostAndPort address) { + var service = createService(address); + log.atInfo() + .with(idKey, serviceId) + .log("advertising service in service discovery"); + client.agentServiceRegister(service); + return this::remove; + } + + private void remove() { + client.agentServiceDeregister(serviceId); + log.atInfo() + .with(idKey, serviceId) + .log("removing service discovery advertisement"); + } + + private NewService createService(HostAndPort address) { + var service = new NewService(); + service.setId(serviceId); + service.setPort(address.getPort()); + service.setAddress(address.getHost()); + service.setCheck(createCheck(address)); + return service; + } + + private static final String updateInterval = "10s"; + + private NewService.Check createCheck(HostAndPort address) { + var check = new NewService.Check(); + check.setGrpc(address.toString()); + check.setInterval(updateInterval); + return check; + } + + @Override + public String toString() { + return "ConsulServiceAdvertisement(serviceId=%s,client=%s)" + .formatted(serviceId, client); + } +} \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/runtime/discovery/ServiceAdvertisement.java b/runtime/src/main/java/jsheets/runtime/discovery/ServiceAdvertisement.java new file mode 100644 index 0000000..56e19dd --- /dev/null +++ b/runtime/src/main/java/jsheets/runtime/discovery/ServiceAdvertisement.java @@ -0,0 +1,11 @@ +package jsheets.runtime.discovery; + +/** + * Represents an existing entry in a service discovery backend. + */ +public interface ServiceAdvertisement { + /** + * Deletes the entry from the discovery backend. + */ + void remove(); +} diff --git a/runtime/src/main/java/jsheets/runtime/discovery/ServiceAdvertisementChannel.java b/runtime/src/main/java/jsheets/runtime/discovery/ServiceAdvertisementChannel.java new file mode 100644 index 0000000..bda28e9 --- /dev/null +++ b/runtime/src/main/java/jsheets/runtime/discovery/ServiceAdvertisementChannel.java @@ -0,0 +1,7 @@ +package jsheets.runtime.discovery; + +import com.google.common.net.HostAndPort; + +public interface ServiceAdvertisementChannel { + ServiceAdvertisement advertise(HostAndPort address); +} diff --git a/runtime/src/main/java/jsheets/runtime/evaluation/EvaluationModule.java b/runtime/src/main/java/jsheets/runtime/evaluation/EvaluationModule.java new file mode 100644 index 0000000..a289558 --- /dev/null +++ b/runtime/src/main/java/jsheets/runtime/evaluation/EvaluationModule.java @@ -0,0 +1,53 @@ +package jsheets.runtime.evaluation; + +import com.google.inject.AbstractModule; +import com.google.inject.Provides; +import com.google.inject.Singleton; + +import javax.inject.Named; +import jsheets.evaluation.EvaluationEngine; +import jsheets.evaluation.shell.ShellEvaluationEngine; +import jsheets.evaluation.shell.execution.ExecutionMethod; +import jsheets.evaluation.shell.execution.ExhaustiveExecution; +import jsheets.config.Config; + +import static jsheets.runtime.evaluation.SandboxConfigSource.accessGraphKey; +import static jsheets.runtime.evaluation.SandboxConfigSource.disableSandboxKey; + +public final class EvaluationModule extends AbstractModule { + public static EvaluationModule create() { + return new EvaluationModule(); + } + + private EvaluationModule() {} + + @Provides + @Singleton + EvaluationEngine evaluationEngine(ExecutionMethod.Factory executionMethodFactory) { + return ShellEvaluationEngine.newBuilder() + .useExecutionMethodFactory(executionMethodFactory) + .create(); + } + + @Provides + @Singleton + ExecutionMethod.Factory executionMethodFactory( + Config config, + @Named("underlyingExecutionMethod") + ExecutionMethod.Factory underlyingExecutionMethodFactory + ) { + boolean disableSandbox = + disableSandboxKey().in(config).orNone().orElse(false); + var accessGraphConfig = accessGraphKey().in(config).require(); + if (disableSandbox) { + return underlyingExecutionMethodFactory; + } + return underlyingExecutionMethodFactory; + } + + + @Named("underlyingExecutionMethod") + ExecutionMethod.Factory underlyingExecutionMethodFactory(Config config) { + return ExhaustiveExecution::create; + } +} \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/runtime/evaluation/SandboxConfigSource.java b/runtime/src/main/java/jsheets/runtime/evaluation/SandboxConfigSource.java new file mode 100644 index 0000000..b4c0a1d --- /dev/null +++ b/runtime/src/main/java/jsheets/runtime/evaluation/SandboxConfigSource.java @@ -0,0 +1,69 @@ +package jsheets.runtime.evaluation; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.util.Optional; + +import com.google.common.flogger.FluentLogger; + +import jsheets.config.Config; +import jsheets.config.RawConfig; + +/** + * Reads the {@code AccessGraph} configuration from the classpath. + */ +public final class SandboxConfigSource implements Config.Source { + private static final FluentLogger log = FluentLogger.forEnclosingClass(); + + private static final Config.Key disableSandboxKey = + Config.Key.ofFlag("evaluation.sandbox.disable"); + + /** + * Since the sandbox is an important security measure, it has to be + * disabled explicitly. + */ + public static Config.Key disableSandboxKey() { + return disableSandboxKey; + } + + private static final Config.Key accessGraphKey = + Config.Key.ofString("evaluation.sandbox.accessGraph"); + + public static Config.Key accessGraphKey() { + return accessGraphKey; + } + + public static SandboxConfigSource fromClassPath() { + return new SandboxConfigSource(); + } + + private SandboxConfigSource() {} + + @Override + public Config load() { + var config = RawConfig.newBuilder(); + readAccessGraphFile() + .ifPresent(accessGraph -> config.with(accessGraphKey, accessGraph)); + return config.create(); + } + + private static final String accessGraphFilePath = + "runtime/evaluation/sandbox/accessGraph.txt"; + + private Optional readAccessGraphFile() { + var resources = Thread.currentThread().getContextClassLoader(); + var file = resources.getResourceAsStream(accessGraphFilePath); + if (file == null) { + log.atConfig().log("could not find %s in classpath", accessGraphFilePath); + return Optional.empty(); + } + try (var input = new BufferedInputStream(file)) { + return Optional.of(new String(input.readAllBytes())); + } catch (IOException failedRead) { + log.atWarning() + .withCause(failedRead) + .log("failed to read accessGraph.txt"); + } + return Optional.empty(); + } +} \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/sandbox/validation/Analysis.java b/runtime/src/main/java/jsheets/sandbox/validation/Analysis.java deleted file mode 100644 index 93c6c56..0000000 --- a/runtime/src/main/java/jsheets/sandbox/validation/Analysis.java +++ /dev/null @@ -1,36 +0,0 @@ -package jsheets.sandbox.validation; - -import java.util.Collection; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.stream.Collectors; - -public final class Analysis { - public static Analysis create() { - return new Analysis(); - } - - public interface Violation {} - - private final Collection violations = - new ConcurrentLinkedQueue<>(); - - private Analysis() {} - - public void report(Violation violation) { - violations.add(violation); - } - - public void reportViolations() { - switch (violations.size()) { - case 0 -> { } - case 1 -> throw new RuntimeException( - violations.iterator().next().toString() - ); - default -> throw new RuntimeException( - violations.stream() - .map(Violation::toString) - .collect(Collectors.joining(", ")) - ); - } - } -} \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/sandbox/validation/Rule.java b/runtime/src/main/java/jsheets/sandbox/validation/Rule.java deleted file mode 100644 index 0a07d04..0000000 --- a/runtime/src/main/java/jsheets/sandbox/validation/Rule.java +++ /dev/null @@ -1,10 +0,0 @@ -package jsheets.sandbox.validation; - -public interface Rule { - record AccessPoint(String className, String methodName) { } - record MethodCall(AccessPoint accessPoint, String owner, String method) { } - record FieldAccess(AccessPoint accessPoint, String owner, String field) { } - - default void visitCall(Analysis analysis, MethodCall call) {} - default void visitFieldAccess(Analysis analysis, FieldAccess access) {} -} diff --git a/runtime/src/main/resources/runtime/evaluation/sandbox/accessGraph.txt b/runtime/src/main/resources/runtime/evaluation/sandbox/accessGraph.txt new file mode 100644 index 0000000..e69de29 diff --git a/runtime/src/test/java/jsheets/config/CamelCaseTest.java b/runtime/src/test/java/jsheets/config/CamelCaseTest.java new file mode 100644 index 0000000..6f7db17 --- /dev/null +++ b/runtime/src/test/java/jsheets/config/CamelCaseTest.java @@ -0,0 +1,53 @@ +package jsheets.config; + +import java.util.function.Function; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import jsheets.config.CamelCase.Case; + +public class CamelCaseTest { + @Test + public void emptyReturnsSameInstance() { + var input = new String(new char[0]); // no interning + Assertions.assertSame(input, CamelCase.convert(input, "_", Case.upper())); + } + + @Test + public void singleCharacterIsConvertedInCase() { + Assertions.assertEquals("a", CamelCase.convert("A", "_", Case.lower())); + Assertions.assertEquals("a", CamelCase.convert("a", "_", Case.lower())); + } + + @Test + public void testToSnakeCase() { + Function toSnakeCase = input -> + CamelCase.convert(input, "_", Case.lower()); + + Assertions.assertEquals("a_bc", toSnakeCase.apply("aBC")); + Assertions.assertEquals("g_rpc", toSnakeCase.apply("gRpc")); + Assertions.assertEquals("g_rpc", toSnakeCase.apply("gRPC")); + Assertions.assertEquals("my_sql", toSnakeCase.apply("MySql")); + Assertions.assertEquals("queen", toSnakeCase.apply("QUEEN")); + Assertions.assertEquals( + "the_sun_is_burning", + toSnakeCase.apply("TheSunIsBurning") + ); + Assertions.assertEquals( + "simon_and_garfunkel", + toSnakeCase.apply("simonAndGarfunkel") + ); + } + + @Test + public void testToScreamingCase() { + Function toScreamingCase = input -> + CamelCase.convert(input, "_", Case.upper()); + + Assertions.assertEquals("LUKE_KELLY", toScreamingCase.apply("LukeKelly")); + Assertions.assertEquals("LUKE_KELLY", toScreamingCase.apply("lukeKelly")); + Assertions.assertEquals("LUKE_KELLY", toScreamingCase.apply("luke_kelly")); + Assertions.assertEquals("ABC", toScreamingCase.apply("abc")); + } +} \ No newline at end of file diff --git a/runtime/src/test/java/jsheets/config/EnvironmentConfigTest.java b/runtime/src/test/java/jsheets/config/EnvironmentConfigTest.java new file mode 100644 index 0000000..ea772f6 --- /dev/null +++ b/runtime/src/test/java/jsheets/config/EnvironmentConfigTest.java @@ -0,0 +1,64 @@ +package jsheets.config; + +import java.util.Map; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class EnvironmentConfigTest { + @Test + public void testLookup() { + var config = new EnvironmentConfig( + "MIRIAM_", + Map.of( + "MIRIAM_TASTE_FAVOURITE_SONG", "A Poem on the Underground Wall", + "MIRIAM_TASTE_FAVOURITE_COLOR", "Teal", + "MIRIAM_TASTE_WORST_MOVIE", "Titanic", + "MIRIAM_PHONE_MAKE", "jMobile" + ) + ); + Assertions.assertEquals( + "A Poem on the Underground Wall", + config.lookup(Config.Key.ofString("taste.favouriteSong")).require() + ); + Assertions.assertEquals( + "jMobile", + config.lookup(Config.Key.ofString("phone.make")).require() + ); + Assertions.assertFalse( + config.lookup(Config.Key.ofString("phone.color")).exists() + ); + Assertions.assertTrue( + config.lookup(Config.Key.ofString("taste.worstMovie")).exists() + ); + Assertions.assertFalse( + config.lookup(Config.Key.ofString("miriam.taste.worstMovie")).exists() + ); + } + + @Test + public void testTranslateKey() { + assertEquals( + "OPINION_THE_BEST_SONG", + EnvironmentConfig.translateKey("opinion.theBestSong") + ); + assertEquals( + "ARTIST_NINA_SIMONE_BLUES", + EnvironmentConfig.translateKey("artist.nina_simone.blues") + ); + assertEquals( + "PLAYBACK_SONG_NAME", + EnvironmentConfig.translateKey("playback.song.name") + ); + assertEquals( + "PLAYBACK_SONG_NAME", + EnvironmentConfig.translateKey("playback.songName") + ); + assertEquals( + "PLAYBACK_SONG_NAME", + EnvironmentConfig.translateKey("playbackSongName") + ); + } +} \ No newline at end of file diff --git a/runtime/src/test/java/jsheets/sandbox/PermissionGraphTest.java b/runtime/src/test/java/jsheets/sandbox/PermissionGraphTest.java deleted file mode 100644 index 8959b5a..0000000 --- a/runtime/src/test/java/jsheets/sandbox/PermissionGraphTest.java +++ /dev/null @@ -1,22 +0,0 @@ -package jsheets.sandbox; - -import jsheets.sandbox.access.AccessGraph; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import static jsheets.sandbox.access.AccessKey.slashSeparated; - -public class PermissionGraphTest { - @Test - public void testDirectPath() { - var graph = AccessGraph.of( - "java.lang", - "java.util.List", - "void java.lang.Thread#sleep()" - ); - Assertions.assertTrue(graph.isPermitted(slashSeparated("java/io"))); - Assertions.assertTrue(graph.isPermitted(slashSeparated("java/lang"))); - Assertions.assertTrue(graph.isPermitted(slashSeparated("java/util/List"))); - Assertions.assertTrue(graph.isPermitted(slashSeparated("java/lang/Thread#sleep()"))); - } -} diff --git a/runtime/src/test/java/jsheets/sandbox/SandboxExecutionTest.java b/runtime/src/test/java/jsheets/sandbox/SandboxExecutionTest.java deleted file mode 100644 index 0102061..0000000 --- a/runtime/src/test/java/jsheets/sandbox/SandboxExecutionTest.java +++ /dev/null @@ -1,36 +0,0 @@ -package jsheets.sandbox; - -import jdk.jshell.JShell; -import jsheets.sandbox.access.AccessGraph; -import jsheets.sandbox.validation.ForbiddenMemberFilter; -import org.junit.jupiter.api.Test; - -import java.util.List; -import java.util.Map; - -public final class SandboxExecutionTest { - @Test - public void test() { - var shell = createSandboxedShell(); - shell.eval(""" - System.out.println("Hello, World!"); - System.err.println("Hello, World!"); - """); - } - - private JShell createSandboxedShell() { - var accessGraph = AccessGraph.of( - "java.lang.Object", - "java.lang.System.out", - "java.io.PrintStream#println" - ); - System.out.println(accessGraph); - return JShell.builder() - .out(System.out) - .err(System.err) - .executionEngine( - SandboxedEnvironment.create(List.of(ForbiddenMemberFilter.create(accessGraph))), - Map.of() - ).build(); - } -} diff --git a/server/build.gradle b/server/build.gradle index c969070..5d49513 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -14,10 +14,10 @@ sourceCompatibility = 16 targetCompatibility = 16 ext { - mongoDbDriverVersion = '4.3.0' + mongoDbDriverVersion = '4.3.2' cfg4jVersion = '4.4.1' slf4jVersion = '1.7.32' - javalinVersion = '4.0.0' + javalinVersion = '4.1.1' argsParseVersion = '0.9.0' javaxAnnotationVersion = '1.3.2' guiceVersion = '5.0.1' @@ -25,7 +25,7 @@ ext { dependencies { implementation project(':protocol') - implementation project(':runtime') + implementation project(':evaluation') implementation "io.micrometer:micrometer-core:$micrometerVersion" implementation "org.mongodb:mongodb-driver-sync:$mongoDbDriverVersion" implementation "org.cfg4j:cfg4j-core:$cfg4jVersion" diff --git a/server/src/main/java/jsheets/server/evaluation/EvaluationConnection.java b/server/src/main/java/jsheets/server/evaluation/EvaluationConnection.java index 7acd064..6c20899 100644 --- a/server/src/main/java/jsheets/server/evaluation/EvaluationConnection.java +++ b/server/src/main/java/jsheets/server/evaluation/EvaluationConnection.java @@ -6,8 +6,8 @@ import io.javalin.websocket.*; import jsheets.*; -import jsheets.runtime.evaluation.Evaluation; -import jsheets.runtime.evaluation.EvaluationEngine; +import jsheets.evaluation.Evaluation; +import jsheets.evaluation.EvaluationEngine; import org.eclipse.jetty.http.HttpStatus; import org.eclipse.jetty.websocket.api.CloseStatus; import org.eclipse.jetty.websocket.api.Session; diff --git a/server/src/main/java/jsheets/server/evaluation/EvaluationModule.java b/server/src/main/java/jsheets/server/evaluation/EvaluationModule.java index 0fac756..1ada105 100644 --- a/server/src/main/java/jsheets/server/evaluation/EvaluationModule.java +++ b/server/src/main/java/jsheets/server/evaluation/EvaluationModule.java @@ -7,8 +7,8 @@ import com.google.inject.Provides; import com.google.inject.Singleton; -import jsheets.runtime.evaluation.EvaluationEngine; -import jsheets.runtime.evaluation.shell.ShellEvaluationEngine; +import jsheets.evaluation.EvaluationEngine; +import jsheets.evaluation.shell.ShellEvaluationEngine; public final class EvaluationModule extends AbstractModule { diff --git a/settings.gradle b/settings.gradle index f4307c3..83749f2 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,7 +1,8 @@ rootProject.name = 'jsheets' +include 'evaluation' include 'runtime' include 'protocol' -include 'server' include 'website' +include 'server' From d93516b95164d6a271ba99693d8f395dd21003d8 Mon Sep 17 00:00:00 2001 From: Merlin Osayimwen Date: Fri, 15 Oct 2021 01:06:51 +0200 Subject: [PATCH 05/14] Cleanup inconsistencies --- evaluation/src/main/resources/.sandbox | 3 --- .../jsheets/evaluation/sandbox/access/AccessGraphTest.java | 2 +- .../src/test/resources/{access-graph.txt => accessGraph.txt} | 0 .../main/java/jsheets/runtime/discovery/AdvertisementHook.java | 2 +- .../main/java/jsheets/runtime/evaluation/EvaluationModule.java | 1 - 5 files changed, 2 insertions(+), 6 deletions(-) delete mode 100644 evaluation/src/main/resources/.sandbox rename evaluation/src/test/resources/{access-graph.txt => accessGraph.txt} (100%) diff --git a/evaluation/src/main/resources/.sandbox b/evaluation/src/main/resources/.sandbox deleted file mode 100644 index 991b958..0000000 --- a/evaluation/src/main/resources/.sandbox +++ /dev/null @@ -1,3 +0,0 @@ -java.lang.* -java.lang.String -! void java.lang.String#intern() diff --git a/evaluation/src/test/java/jsheets/evaluation/sandbox/access/AccessGraphTest.java b/evaluation/src/test/java/jsheets/evaluation/sandbox/access/AccessGraphTest.java index 9c3a584..56b733b 100644 --- a/evaluation/src/test/java/jsheets/evaluation/sandbox/access/AccessGraphTest.java +++ b/evaluation/src/test/java/jsheets/evaluation/sandbox/access/AccessGraphTest.java @@ -14,7 +14,7 @@ public class AccessGraphTest { @Test public void testReadingFromFile() { - var specification = readFileContent("access-graph.txt"); + var specification = readFileContent("accessGraph.txt"); var graph = AccessGraph.of(specification.split("\n")); Assertions.assertFalse(graph.isPermitted(dotSeparated("a.b.c"))); Assertions.assertTrue(graph.isPermitted(dotSeparated("a.b.c.Foo"))); diff --git a/evaluation/src/test/resources/access-graph.txt b/evaluation/src/test/resources/accessGraph.txt similarity index 100% rename from evaluation/src/test/resources/access-graph.txt rename to evaluation/src/test/resources/accessGraph.txt diff --git a/runtime/src/main/java/jsheets/runtime/discovery/AdvertisementHook.java b/runtime/src/main/java/jsheets/runtime/discovery/AdvertisementHook.java index 6c54647..b968754 100644 --- a/runtime/src/main/java/jsheets/runtime/discovery/AdvertisementHook.java +++ b/runtime/src/main/java/jsheets/runtime/discovery/AdvertisementHook.java @@ -17,9 +17,9 @@ public static AdvertisementHook create( return new AdvertisementHook(advertisedHost, channel); } - private volatile ServiceAdvertisement advertisement; private final HostAndPort advertisedHost; private final ServiceAdvertisementChannel advertisementChannel; + private volatile ServiceAdvertisement advertisement; @Inject AdvertisementHook( diff --git a/runtime/src/main/java/jsheets/runtime/evaluation/EvaluationModule.java b/runtime/src/main/java/jsheets/runtime/evaluation/EvaluationModule.java index a289558..256b92d 100644 --- a/runtime/src/main/java/jsheets/runtime/evaluation/EvaluationModule.java +++ b/runtime/src/main/java/jsheets/runtime/evaluation/EvaluationModule.java @@ -45,7 +45,6 @@ ExecutionMethod.Factory executionMethodFactory( return underlyingExecutionMethodFactory; } - @Named("underlyingExecutionMethod") ExecutionMethod.Factory underlyingExecutionMethodFactory(Config config) { return ExhaustiveExecution::create; From 5e59e1179f5f0c890a3ea52e829f28580ef2f312 Mon Sep 17 00:00:00 2001 From: Merlin Osayimwen Date: Fri, 15 Oct 2021 01:55:06 +0200 Subject: [PATCH 06/14] Explain runtime --- runtime/README.md | 130 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 runtime/README.md diff --git a/runtime/README.md b/runtime/README.md new file mode 100644 index 0000000..058805b --- /dev/null +++ b/runtime/README.md @@ -0,0 +1,130 @@ +# Runtime +The runtime component is responsible for evaluating Snippets, +it exposes the +[SnippetRuntime](../protocol/src/main/proto/jsheets/api/snippet_runtime.proto) +as a [gRpc Service](https://grpc.io). +Snippets are currently evaluated by an instrumented version of +[JShell](https://docs.oracle.com/javase/9/jshell/introduction-jshell.htm) that +is modified for script-like code and secure execution. Apart from the active +*JShell* instances, the *Runtime* is **Stateless**, allowing it to be scaled +horizontally. This is beneficial, as user code evaluation poses a high chance +for *JVM crashes*. + +### Sandboxing +The JVM itself provides a sufficient sandbox. If we restrict the methods +that can be called to that of classes without side effects to the system and +JVM itself and prevent `java.lang.reflect` and `java.lang.invoke`, code can +barely do any direct harm (other than using too many resources). + +The [evaluation](../evaluation) module provides the +`jsheets.evaluation.sandbox.access` library, that can be used to restrict +access to a given list of methods, fields and classes. It is configured +using a text file that looks like a `.gitignore` file. + +Example: +``` +java.util.List +java.util.Collection +java.lang.Thread#currentThread +!java.lang.Object#wait +``` + +Method signatures can be written as follows: + +- `java.lang.Object#equals(java.lang.Object):boolean` is a full signature with +parameter list and return type. It only matches methods that have the exact +same class name, name, parameter and return types. + +- `java.lang.Object#equals(*):boolean` or `java.lang.Object#equals:boolean` +has a wildcard parameter list. It matches any method that has the same class +name, name and return type. This is especially useful for methods with many +overloads. + +- `java.lang.Object#equals(java.lang.Object):*` or +`java.lang.Object#equals(java.lang.Object)` has a wildcard return type. It +matches any method that has the same class name, name and parameter types. + +- `java.lang.Object#equals:*` or `java.lang.Object#equals` has a wildcard +parameter list and return type. It matches any method that has the same class +name and name. + +For example, if we have the following class `Library` in package `evilcorp.coolib` +```java +package evilcorp.coolib; + +class Library { + int count(int[] integers) { ... } + + int count(double[] doubles) { ... } + + int count(float[] floats) { ... } + + long count(int[][] twoDimensionIntegers) { ... } + + void quit() { + System.exit(-1); + } + + void quit(String message) { + System.err.println(message); + System.exit(-1); + } +} +``` + +we can write following access graph configs + +``` +evilcorp.coolib.Library#count(*)* +``` +Here we just enable the methods that we wish to call, but this would be +cumbersome, if we had to do it for every method in a big library. +Instead, we want to exclude the methods that are not allowed: +``` +evilcorp.coolib +!evilcorp.coolib.Library#quit +``` +Now every class within `evilcorp.coolib` and every of their methods are +allowed, with exception to any method in `evilcorp.coolib.Library` that +is named `quit`. + +If this sounds too out of context to you, picture the `java.lang.System` class, +which is a very central and useful class in java's standard library, it contains +fields and methods that are essential to some programs that do not pose any +security risk (like `System#identityHashCode(Object)`), but also methods like +`System#exit(int)`, we would thus be very careful with granting access to +this class. + + + + +### Scaling +Since the *runtime* does not save any data and its state only consists of +the active evaluations, it can be scaled horizontally to **thousands** of +instances. It is important to keep the evaluations per instance fairly low, +this reduces the amount of evaluations that are affected by crashes and +helps not to overuse resources (such as processors and memory). + + +### Handling Crashes +If the *runtime* crashes due to bad code being evaluated, it is taken out of the +service discovery and will not receive any further requests from the *backend*. +Current evaluations will time out and the backend can choose to retry them +or report an error to the client. Given the deployment strategy, the instance +may be recreated immediately afterwards and put back into the service discovery. + +### Future Planning + +The current evaluation model is limited: +- Code is restricted (in what libraries and methods to use) +to prevent security issues and simulate a sandbox. +- Crashes of individual evaluations result in a crash of the entire instance, +thus preempting all other evaluations. + +But it provides a somewhat solid and definitely scalable solution for the +first version of *JShell*. + +Future implementations may spawn [Containers](https://linuxcontainers.org/) +for every *Sheet* or *Snippet* or *User*. Some features could also be +limited to those who have a *user account*. This container could load some kind +of state, provide access to a small (possibly persistent) file system and more. \ No newline at end of file From 999c57db1af10e7c7614c2a0caa24772c081a880 Mon Sep 17 00:00:00 2001 From: Merlin Osayimwen Date: Fri, 15 Oct 2021 02:06:55 +0200 Subject: [PATCH 07/14] Improve runtime module documentation --- runtime/README.md | 44 ++++++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/runtime/README.md b/runtime/README.md index 058805b..cd801a0 100644 --- a/runtime/README.md +++ b/runtime/README.md @@ -5,23 +5,23 @@ it exposes the as a [gRpc Service](https://grpc.io). Snippets are currently evaluated by an instrumented version of [JShell](https://docs.oracle.com/javase/9/jshell/introduction-jshell.htm) that -is modified for script-like code and secure execution. Apart from the active -*JShell* instances, the *Runtime* is **Stateless**, allowing it to be scaled -horizontally. This is beneficial, as user code evaluation poses a high chance +is modified for secure execution of script-like code. +Apart from the active *JShell* instances, the *Runtime* is **Stateless**, +allowing it to be scaled +horizontally. This is beneficial, as user code evaluation poses a chance for *JVM crashes*. ### Sandboxing -The JVM itself provides a sufficient sandbox. If we restrict the methods -that can be called to that of classes without side effects to the system and -JVM itself and prevent `java.lang.reflect` and `java.lang.invoke`, code can +The JVM itself is a sufficient sandbox, if we restrict the methods +that can be called to that of classes without side effects to the system +and prevent `java.lang.reflect` and `java.lang.invoke`, code can barely do any direct harm (other than using too many resources). The [evaluation](../evaluation) module provides the -`jsheets.evaluation.sandbox.access` library, that can be used to restrict +`jsheets.evaluation.sandbox.access` library, that is used to restrict access to a given list of methods, fields and classes. It is configured -using a text file that looks like a `.gitignore` file. +using a text file that looks similar to a `.gitignore`: -Example: ``` java.util.List java.util.Collection @@ -29,6 +29,7 @@ java.lang.Thread#currentThread !java.lang.Object#wait ``` +#### Format Method signatures can be written as follows: - `java.lang.Object#equals(java.lang.Object):boolean` is a full signature with @@ -48,7 +49,8 @@ matches any method that has the same class name, name and parameter types. parameter list and return type. It matches any method that has the same class name and name. -For example, if we have the following class `Library` in package `evilcorp.coolib` +#### Example +Given the following class `Library` in package `evilcorp.coolib` ```java package evilcorp.coolib; @@ -72,7 +74,7 @@ class Library { } ``` -we can write following access graph configs +we can write the following access graph configs: ``` evilcorp.coolib.Library#count(*)* @@ -101,22 +103,24 @@ this class. ### Scaling Since the *runtime* does not save any data and its state only consists of the active evaluations, it can be scaled horizontally to **thousands** of -instances. It is important to keep the evaluations per instance fairly low, -this reduces the amount of evaluations that are affected by crashes and -helps not to overuse resources (such as processors and memory). +instances. + +It is important to keep the evaluations per instance fairly low to +reduce the amount of evaluations that are affected by crashes and +lower usage of system resources (such as processors and memory). ### Handling Crashes -If the *runtime* crashes due to bad code being evaluated, it is taken out of the -service discovery and will not receive any further requests from the *backend*. -Current evaluations will time out and the backend can choose to retry them -or report an error to the client. Given the deployment strategy, the instance -may be recreated immediately afterwards and put back into the service discovery. +If the *runtime* crashes, it is taken out of the service discovery and will not +receive any further requests from the *backend*. Current evaluations will time +out and the backend can choose to retry them or report an error to the client. +Given the deployment strategy, the instance may be recreated immediately +afterwards and put back into the service discovery. ### Future Planning The current evaluation model is limited: -- Code is restricted (in what libraries and methods to use) +- Code is restricted (in what libraries and methods it uses) to prevent security issues and simulate a sandbox. - Crashes of individual evaluations result in a crash of the entire instance, thus preempting all other evaluations. From d01c8724887b749e7ce194431198abf8c796e43a Mon Sep 17 00:00:00 2001 From: Merlin Osayimwen Date: Fri, 15 Oct 2021 02:29:15 +0200 Subject: [PATCH 08/14] Add initial runtime client --- build.gradle | 2 +- runtime/build.gradle | 1 - server/build.gradle | 4 +- .../client/ConsulServiceDiscovery.java | 19 ++++ .../server/evaluation/client/EnginePool.java | 9 ++ .../client/SnippetRuntimeEngine.java | 100 ++++++++++++++++++ 6 files changed, 132 insertions(+), 3 deletions(-) create mode 100644 server/src/main/java/jsheets/server/evaluation/client/ConsulServiceDiscovery.java create mode 100644 server/src/main/java/jsheets/server/evaluation/client/EnginePool.java create mode 100644 server/src/main/java/jsheets/server/evaluation/client/SnippetRuntimeEngine.java diff --git a/build.gradle b/build.gradle index 824fce8..0248c5e 100644 --- a/build.gradle +++ b/build.gradle @@ -12,5 +12,5 @@ ext { micrometerVersion = '1.7.4' consuleClientVersion = '1.4.5' javaxAnnotationVersion = '1.3.2' - recordBuilderVersion = '26' + recordBuilderVersion = '28' } \ No newline at end of file diff --git a/runtime/build.gradle b/runtime/build.gradle index 758dec9..644fa29 100644 --- a/runtime/build.gradle +++ b/runtime/build.gradle @@ -28,7 +28,6 @@ dependencies { implementation "com.ecwid.consul:consul-api:$consuleClientVersion" implementation "io.micrometer:micrometer-core:$micrometerVersion" implementation "org.mongodb:mongodb-driver-sync:$mongoDbDriverVersion" - implementation "org.cfg4j:cfg4j-core:$cfg4jVersion" implementation "org.slf4j:slf4j-simple:$slf4jVersion" implementation "io.grpc:grpc-all:$grpcVersion" implementation "com.google.protobuf:protobuf-java-util:$protobufJavaVersion" diff --git a/server/build.gradle b/server/build.gradle index 5d49513..f22bafb 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -28,7 +28,7 @@ dependencies { implementation project(':evaluation') implementation "io.micrometer:micrometer-core:$micrometerVersion" implementation "org.mongodb:mongodb-driver-sync:$mongoDbDriverVersion" - implementation "org.cfg4j:cfg4j-core:$cfg4jVersion" + implementation "com.ecwid.consul:consul-api:$consuleClientVersion" implementation "io.javalin:javalin:$javalinVersion" implementation "org.slf4j:slf4j-simple:$slf4jVersion" implementation "com.google.protobuf:protobuf-java-util:$protobufJavaVersion" @@ -37,6 +37,8 @@ dependencies { implementation "com.google.flogger:flogger:$floggerVersion" implementation "com.google.flogger:flogger-slf4j-backend:$floggerVersion" implementation "com.google.inject:guice:$guiceVersion" + implementation "io.grpc:grpc-netty:$grpcVersion" + implementation "io.grpc:grpc-stub:$grpcVersion" testImplementation "org.junit.jupiter:junit-jupiter-api:$junitPlatformVersion" testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:$junitPlatformVersion" } diff --git a/server/src/main/java/jsheets/server/evaluation/client/ConsulServiceDiscovery.java b/server/src/main/java/jsheets/server/evaluation/client/ConsulServiceDiscovery.java new file mode 100644 index 0000000..0a8fc7d --- /dev/null +++ b/server/src/main/java/jsheets/server/evaluation/client/ConsulServiceDiscovery.java @@ -0,0 +1,19 @@ +package jsheets.server.evaluation.client; + +import java.util.Optional; + +import com.ecwid.consul.v1.ConsulClient; +import jsheets.evaluation.EvaluationEngine; + +public final class ConsulServiceDiscovery implements EnginePool { + private final ConsulClient client; + + private ConsulServiceDiscovery(ConsulClient client) { + this.client = client; + } + + @Override + public Optional select() { + return Optional.empty(); + } +} \ No newline at end of file diff --git a/server/src/main/java/jsheets/server/evaluation/client/EnginePool.java b/server/src/main/java/jsheets/server/evaluation/client/EnginePool.java new file mode 100644 index 0000000..4cd7224 --- /dev/null +++ b/server/src/main/java/jsheets/server/evaluation/client/EnginePool.java @@ -0,0 +1,9 @@ +package jsheets.server.evaluation.client; + +import java.util.Optional; + +import jsheets.evaluation.EvaluationEngine; + +public interface EnginePool { + Optional select(); +} diff --git a/server/src/main/java/jsheets/server/evaluation/client/SnippetRuntimeEngine.java b/server/src/main/java/jsheets/server/evaluation/client/SnippetRuntimeEngine.java new file mode 100644 index 0000000..edcc68f --- /dev/null +++ b/server/src/main/java/jsheets/server/evaluation/client/SnippetRuntimeEngine.java @@ -0,0 +1,100 @@ +package jsheets.server.evaluation.client; + +import java.util.Objects; + +import com.google.common.flogger.FluentLogger; +import com.google.common.flogger.MetadataKey; + +import io.grpc.Channel; +import io.grpc.stub.StreamObserver; + +import jsheets.EvaluateRequest; +import jsheets.EvaluateResponse; +import jsheets.SnippetRuntimeGrpc; +import jsheets.SnippetRuntimeGrpc.SnippetRuntimeStub; +import jsheets.StartEvaluationRequest; +import jsheets.StopEvaluationRequest; +import jsheets.evaluation.Evaluation; +import jsheets.evaluation.EvaluationEngine; + +/** + * Client side {@link EvaluationEngine} that connects to a + * {@code SnippetRuntime} to evaluate snippets. + */ +final class SnippetRuntimeEngine implements EvaluationEngine { + private static final FluentLogger log = FluentLogger.forEnclosingClass(); + + public static SnippetRuntimeEngine forChannel(Channel channel) { + Objects.requireNonNull(channel, "channel"); + return new SnippetRuntimeEngine(SnippetRuntimeGrpc.newStub(channel)); + } + + private final SnippetRuntimeStub client; + + private SnippetRuntimeEngine(SnippetRuntimeStub client) { + this.client = client; + } + + @Override + public Evaluation start( + StartEvaluationRequest request, + Evaluation.Listener listener + ) { + var snippetId = request.getSnippet().getReference().getSnippetId(); + var observer = new ListenerBoundObserver(snippetId, listener); + var call = client.evaluate(observer); + call.onNext(wrapStartRequest(request)); + return () -> call.onNext(createStopRequest()); + } + + private EvaluateRequest wrapStartRequest(StartEvaluationRequest request) { + return EvaluateRequest.newBuilder() + .setStart(request) + .build(); + } + + private EvaluateRequest createStopRequest() { + return EvaluateRequest.newBuilder() + .setStop(StopEvaluationRequest.getDefaultInstance()) + .build(); + } + + static final class ListenerBoundObserver implements StreamObserver { + private final String snippetId; + private final Evaluation.Listener listener; + + private ListenerBoundObserver( + String snippetId, + Evaluation.Listener listener + ) { + this.snippetId = snippetId; + this.listener = listener; + } + + @Override + public void onNext(EvaluateResponse response) { + listener.send(response); + } + + @Override + public void onCompleted() { + listener.close(); + } + + private static final MetadataKey snippetIdKey = + MetadataKey.single("snippetId", String.class); + + @Override + public void onError(Throwable failure) { + log.atWarning() + .withCause(failure) + .with(snippetIdKey, snippetId) + .log("received error response in evaluate call"); + } + } + + @Override + public String toString() { + return "SnippetRuntimeEngine(client=%s)".formatted(client); + } +} \ No newline at end of file From d00434ec8023c496a167e4bf26554ed22bed4bbe Mon Sep 17 00:00:00 2001 From: Merlin Osayimwen Date: Fri, 15 Oct 2021 18:25:09 +0200 Subject: [PATCH 09/14] Fix execution and clean up code/documentation --- deploy/kubernetes/namespace.yml | 4 ++ deploy/kubernetes/server.yml | 56 ++++++++++++++++++ deploy/minimal/README.md | 5 ++ .../minimal/docker-compose.yml | 0 .../evaluation/failure/FailedEvaluation.java | 21 +++++++ .../evaluation/sandbox/SandboxLoader.java | 7 ++- .../sandbox/access/MethodSignature.java | 6 ++ .../sandbox/validation/Analysis.java | 14 ++++- .../validation/ForbiddenMemberFilter.java | 51 ++++++++++++++++- .../evaluation/shell/ShellEvaluation.java | 57 ++++++++++++++----- .../shell/ShellEvaluationEngine.java | 13 ----- .../environment}/SandboxedEnvironment.java | 27 ++++++--- .../shell/execution/ExhaustiveExecution.java | 7 ++- .../sandbox/SandboxExecutionTest.java | 1 + .../shell/ExhaustiveExecutionTest.java | 1 - runtime/README.md | 38 ++++++++++--- .../java/jsheets/runtime/ConfigModule.java | 2 +- .../jsheets/runtime/ServerSetupModule.java | 12 ++-- .../runtime/evaluation/EvaluationModule.java | 35 +++++++----- .../runtime/evaluation/defaultImports.txt | 7 +++ .../evaluation/sandbox/accessGraph.txt | 2 + 21 files changed, 292 insertions(+), 74 deletions(-) create mode 100644 deploy/kubernetes/namespace.yml create mode 100644 deploy/kubernetes/server.yml create mode 100644 deploy/minimal/README.md rename docker-compose.yml => deploy/minimal/docker-compose.yml (100%) create mode 100644 evaluation/src/main/java/jsheets/evaluation/failure/FailedEvaluation.java rename evaluation/src/main/java/jsheets/evaluation/{sandbox => shell/environment}/SandboxedEnvironment.java (50%) create mode 100644 runtime/src/main/resources/runtime/evaluation/defaultImports.txt diff --git a/deploy/kubernetes/namespace.yml b/deploy/kubernetes/namespace.yml new file mode 100644 index 0000000..222d8b7 --- /dev/null +++ b/deploy/kubernetes/namespace.yml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: jsheets \ No newline at end of file diff --git a/deploy/kubernetes/server.yml b/deploy/kubernetes/server.yml new file mode 100644 index 0000000..ba9f25e --- /dev/null +++ b/deploy/kubernetes/server.yml @@ -0,0 +1,56 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: server + namespace: jsheets + labels: + app: server +spec: + selector: + matchLabels: + app: server + replicas: 3 + template: + metadata: + name: server + labels: + app: server + spec: + containers: + - name: server + image: ehenoma/jsheets:latest + ports: + - containerPort: 8080 + hostPort: 8080 + env: + - name: JSHEETS_SERVER_PORT + value: "8080" +--- +apiVersion: v1 +kind: Service +metadata: + name: server-service + namespace: jsheets +spec: + selector: + app: server + clusterIP: None + ports: + - name: http + port: 8080 + targetPort: 8080 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: server-ingress + namespace: jsheets +spec: + rules: + - http: + paths: + - path: "/" + pathType: Prefix + backend: + service: + name: server-service diff --git a/deploy/minimal/README.md b/deploy/minimal/README.md new file mode 100644 index 0000000..3350775 --- /dev/null +++ b/deploy/minimal/README.md @@ -0,0 +1,5 @@ +# Minimal Deployment + +Contains the server itself and a document store. +The server is responsible for evaluating snippets, which is not +ideal for public/highly frequented websites. \ No newline at end of file diff --git a/docker-compose.yml b/deploy/minimal/docker-compose.yml similarity index 100% rename from docker-compose.yml rename to deploy/minimal/docker-compose.yml diff --git a/evaluation/src/main/java/jsheets/evaluation/failure/FailedEvaluation.java b/evaluation/src/main/java/jsheets/evaluation/failure/FailedEvaluation.java new file mode 100644 index 0000000..b9fe030 --- /dev/null +++ b/evaluation/src/main/java/jsheets/evaluation/failure/FailedEvaluation.java @@ -0,0 +1,21 @@ +package jsheets.evaluation.failure; + +import jsheets.EvaluationError; + +import java.util.Locale; +import java.util.Optional; +import java.util.stream.Stream; + +public interface FailedEvaluation { + Stream describe(Locale locale); + + static Optional capture(Throwable failure) { + while (failure != null) { + if (failure instanceof FailedEvaluation) { + return Optional.of((FailedEvaluation) failure); + } + failure = failure.getCause(); + } + return Optional.empty(); + } +} diff --git a/evaluation/src/main/java/jsheets/evaluation/sandbox/SandboxLoader.java b/evaluation/src/main/java/jsheets/evaluation/sandbox/SandboxLoader.java index 6957bf6..9ceb117 100644 --- a/evaluation/src/main/java/jsheets/evaluation/sandbox/SandboxLoader.java +++ b/evaluation/src/main/java/jsheets/evaluation/sandbox/SandboxLoader.java @@ -47,8 +47,11 @@ private SandboxLoader(Collection rules) { this.rules = rules; } - public void install() { - Thread.currentThread().setContextClassLoader(loader); + public Runnable install() { + var thread = Thread.currentThread(); + var previousLoader = thread.getContextClassLoader(); + thread.setContextClassLoader(loader); + return () -> thread.setContextClassLoader(previousLoader); } @Override diff --git a/evaluation/src/main/java/jsheets/evaluation/sandbox/access/MethodSignature.java b/evaluation/src/main/java/jsheets/evaluation/sandbox/access/MethodSignature.java index 02b282d..bc07041 100644 --- a/evaluation/src/main/java/jsheets/evaluation/sandbox/access/MethodSignature.java +++ b/evaluation/src/main/java/jsheets/evaluation/sandbox/access/MethodSignature.java @@ -97,6 +97,12 @@ public String format() { ); } + private static final String constructorName = ""; + + public boolean isConstructor() { + return methodName.equals(constructorName); + } + public String formatNameAndParameters() { return "%s(%s)".formatted(methodName, String.join(", ", parameterTypes)); } diff --git a/evaluation/src/main/java/jsheets/evaluation/sandbox/validation/Analysis.java b/evaluation/src/main/java/jsheets/evaluation/sandbox/validation/Analysis.java index 5771aa4..c8e596e 100644 --- a/evaluation/src/main/java/jsheets/evaluation/sandbox/validation/Analysis.java +++ b/evaluation/src/main/java/jsheets/evaluation/sandbox/validation/Analysis.java @@ -1,6 +1,10 @@ package jsheets.evaluation.sandbox.validation; +import jsheets.EvaluationError; +import jsheets.evaluation.failure.FailedEvaluation; + import java.util.Collection; +import java.util.Locale; import java.util.Set; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.stream.Collectors; @@ -11,7 +15,7 @@ public static Analysis create() { return new Analysis(); } - public interface Violation {} + public interface Violation extends FailedEvaluation {} private final Collection violations = new ConcurrentLinkedQueue<>(); @@ -28,7 +32,7 @@ public void reportViolations() { } } - static final class FailedAnalysis extends RuntimeException { + static final class FailedAnalysis extends RuntimeException implements FailedEvaluation { private final Collection violations; private FailedAnalysis(Collection violations) { @@ -43,6 +47,12 @@ private FailedAnalysis(Collection violations) { public Stream violations() { return violations.stream(); } + + @Override + public Stream describe(Locale locale) { + return violations.stream() + .flatMap(violation -> violation.describe(locale)); + } } public static Stream captureViolations(Throwable failure) { diff --git a/evaluation/src/main/java/jsheets/evaluation/sandbox/validation/ForbiddenMemberFilter.java b/evaluation/src/main/java/jsheets/evaluation/sandbox/validation/ForbiddenMemberFilter.java index f8d73cf..975696b 100644 --- a/evaluation/src/main/java/jsheets/evaluation/sandbox/validation/ForbiddenMemberFilter.java +++ b/evaluation/src/main/java/jsheets/evaluation/sandbox/validation/ForbiddenMemberFilter.java @@ -1,8 +1,11 @@ package jsheets.evaluation.sandbox.validation; import java.util.Arrays; +import java.util.Locale; import java.util.Objects; +import java.util.stream.Stream; +import jsheets.EvaluationError; import jsheets.evaluation.sandbox.access.AccessGraph; import jsheets.evaluation.sandbox.access.AccessKey; import jsheets.evaluation.sandbox.access.MethodSignature; @@ -21,10 +24,35 @@ private ForbiddenMemberFilter(AccessGraph accessGraph) { this.accessGraph = accessGraph; } - public record ForbiddenMethod(MethodSignature method) implements Analysis.Violation {} + public record ForbiddenMethod(MethodSignature method) implements Analysis.Violation { + @Override + public Stream describe(Locale locale) { + var message = method.isConstructor() + ? formatForConstructor(locale) + : formatForMethod(locale); + return Stream.of( + EvaluationError.newBuilder() + .setKind("sandbox") + .setMessage(message) + .build() + ); + } + + private String formatForConstructor(Locale locale) { + return "The class %s is not allowed".formatted(method.className()); + } + + private String formatForMethod(Locale locale) { + return "The method %s in %s is not allowed" + .formatted(method.methodName(), method.className()); + } + } @Override public void visitCall(Analysis analysis, MethodCall call) { + if (isClassExcluded(call.owner())) { + return; + } var signature = createSignatureOfCall(call); if (!accessGraph.isMethodPermitted(signature)) { analysis.report(new ForbiddenMethod(signature)); @@ -43,13 +71,32 @@ private MethodSignature createSignatureOfCall(MethodCall call) { ).build(); } - public record ForbiddenField(String owner, String field) implements Analysis.Violation {} + public record ForbiddenField(String owner, String field) implements Analysis.Violation { + @Override + public Stream describe(Locale locale) { + return Stream.of( + EvaluationError.newBuilder() + .setKind("sandbox") + .setMessage("The field %s in %s is not allowed".formatted(field, owner)) + .build() + ); + } + } @Override public void visitFieldAccess(Analysis analysis, FieldAccess access) { + if (isClassExcluded(access.owner())) { + return; + } var key = "%s.%s".formatted(access.owner(), access.field()); if (!accessGraph.isPermitted(AccessKey.dotSeparated(key))) { analysis.report(new ForbiddenField(access.owner(), access.field())); } } + + private static final String generatedSnippetClassPrefix = "REPL.$JShell$"; + + private boolean isClassExcluded(String className) { + return className.startsWith(generatedSnippetClassPrefix); + } } \ No newline at end of file diff --git a/evaluation/src/main/java/jsheets/evaluation/shell/ShellEvaluation.java b/evaluation/src/main/java/jsheets/evaluation/shell/ShellEvaluation.java index 9382363..c27436d 100644 --- a/evaluation/src/main/java/jsheets/evaluation/shell/ShellEvaluation.java +++ b/evaluation/src/main/java/jsheets/evaluation/shell/ShellEvaluation.java @@ -21,9 +21,9 @@ import jsheets.SnippetSources; import jsheets.StartEvaluationRequest; import jsheets.evaluation.Evaluation; +import jsheets.evaluation.failure.FailedEvaluation; import jsheets.evaluation.shell.environment.ExecutionEnvironment; import jsheets.evaluation.shell.execution.ExecutionMethod; -import jsheets.source.SharedSources; final class ShellEvaluation implements Evaluation { private enum Stage { Initial, Starting, Evaluating, Terminated } @@ -33,14 +33,12 @@ private enum Stage { Initial, Starting, Evaluating, Terminated } private volatile JShell shell; private volatile ExecutionMethod executionMethod; private final Evaluation.Listener listener; - private final SharedSources sources; private final ExecutionEnvironment environment; private final ExecutionMethod.Factory executionMethodFactory; private final MessageOutput messageOutput; private final AtomicReference stage = new AtomicReference<>(Stage.Initial); ShellEvaluation( - Clock clock, ExecutionEnvironment environment, ExecutionMethod.Factory executionMethodFactory, Evaluation.Listener listener, @@ -50,7 +48,6 @@ private enum Stage { Initial, Starting, Evaluating, Terminated } this.messageOutput = messageOutput; this.executionMethodFactory = executionMethodFactory; this.environment = environment; - sources = SharedSources.createEmpty(clock); } public void start(StartEvaluationRequest request) { @@ -82,19 +79,49 @@ private void evaluateComponent( reportEvent(component.getId(), snippet, response); } messageOutput.flush(); - } catch (Exception failedEvaluation) { - log.atWarning() - .atMostEvery(5, TimeUnit.SECONDS) - .withCause(failedEvaluation).log("shell evaluation failed"); - response.addError( - EvaluationError.newBuilder() - .setComponentId(component.getId()) - .setKind("internal") - .build() - ); + } catch (Throwable failedEvaluation) { + reportError(component, response, failedEvaluation); } } + private void reportError( + SnippetSources.CodeComponent component, + EvaluateResponse.Builder response, + Throwable failure + ) { + FailedEvaluation.capture(failure).ifPresentOrElse( + value -> reportFailedEvaluation(component, response, value), + () -> reportInternalFailure(component, response, failure) + ); + } + + private void reportFailedEvaluation( + SnippetSources.CodeComponent component, + EvaluateResponse.Builder response, + FailedEvaluation failedEvaluation + ) { + failedEvaluation.describe(Locale.ENGLISH) + .distinct() + .map(error -> error.toBuilder().setComponentId(component.getId())) + .forEach(response::addError); + } + + private void reportInternalFailure( + SnippetSources.CodeComponent component, + EvaluateResponse.Builder response, + Throwable failure + ) { + log.atWarning() + .atMostEvery(5, TimeUnit.SECONDS) + .withCause(failure).log("shell evaluation failed"); + response.addError( + EvaluationError.newBuilder() + .setComponentId(component.getId()) + .setKind("internal") + .build() + ); + } + private void reportEvent( String componentId, SnippetEvent event, @@ -108,6 +135,7 @@ private void reportEvent( response.addResult( EvaluationResult.newBuilder() .setComponentId(componentId) + .setKind(EvaluationResult.Kind.INFO) .setOutput(event.value()) .build() ); @@ -159,7 +187,6 @@ private void cleanUp() { public String toString() { return MoreObjects.toStringHelper(this) .add("shell", shell) - .add("sources", sources) .add("environment", environment) .add("stage", stage) .toString(); diff --git a/evaluation/src/main/java/jsheets/evaluation/shell/ShellEvaluationEngine.java b/evaluation/src/main/java/jsheets/evaluation/shell/ShellEvaluationEngine.java index a1b6eea..77da92c 100644 --- a/evaluation/src/main/java/jsheets/evaluation/shell/ShellEvaluationEngine.java +++ b/evaluation/src/main/java/jsheets/evaluation/shell/ShellEvaluationEngine.java @@ -2,7 +2,6 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; -import java.time.Clock; import java.time.Duration; import java.util.Objects; import java.util.concurrent.Executor; @@ -18,7 +17,6 @@ import jsheets.evaluation.shell.execution.SystemBasedExecutionMethodFactory; public final class ShellEvaluationEngine implements EvaluationEngine { - private final Clock clock; private final Executor workerPool; private final ScheduledExecutorService scheduler; private final ExecutionEnvironment executionEnvironment; @@ -26,14 +24,12 @@ public final class ShellEvaluationEngine implements EvaluationEngine { private final Duration messageFlushInterval; private ShellEvaluationEngine( - Clock clock, Executor workerPool, ScheduledExecutorService scheduler, ExecutionEnvironment executionEnvironment, ExecutionMethod.Factory executionMethodFactory, Duration messageFlushInterval ) { - this.clock = clock; this.workerPool = workerPool; this.scheduler = scheduler; this.executionEnvironment = executionEnvironment; @@ -53,7 +49,6 @@ public Evaluation start( private ShellEvaluation createEvaluation(Evaluation.Listener listener) { return new ShellEvaluation( - clock, executionEnvironment, executionMethodFactory, listener, @@ -70,19 +65,12 @@ public static Builder newBuilder() { } public static final class Builder { - private Clock clock = Clock.systemUTC(); private Executor workerPool; private ExecutionEnvironment environment; private Duration messageFlushInterval; private ScheduledExecutorService scheduler; private ExecutionMethod.Factory executionMethodFactory; - public Builder useClock(Clock clock) { - Objects.requireNonNull(clock, "clock"); - this.clock = clock; - return this; - } - public Builder useWorkerPool(Executor pool) { Objects.requireNonNull(pool, "workerPool"); workerPool = pool; @@ -115,7 +103,6 @@ public Builder withMessageFlushInterval(Duration messageFlushInterval) { public EvaluationEngine create() { return new ShellEvaluationEngine( - clock, selectWorkerPool(), selectScheduler(), selectEnvironment(), diff --git a/evaluation/src/main/java/jsheets/evaluation/sandbox/SandboxedEnvironment.java b/evaluation/src/main/java/jsheets/evaluation/shell/environment/SandboxedEnvironment.java similarity index 50% rename from evaluation/src/main/java/jsheets/evaluation/sandbox/SandboxedEnvironment.java rename to evaluation/src/main/java/jsheets/evaluation/shell/environment/SandboxedEnvironment.java index 8830ebd..2abd2d1 100644 --- a/evaluation/src/main/java/jsheets/evaluation/sandbox/SandboxedEnvironment.java +++ b/evaluation/src/main/java/jsheets/evaluation/shell/environment/SandboxedEnvironment.java @@ -1,25 +1,38 @@ -package jsheets.evaluation.sandbox; +package jsheets.evaluation.shell.environment; import jdk.jshell.execution.DirectExecutionControl; import jdk.jshell.spi.ExecutionControl; import jdk.jshell.spi.ExecutionControlProvider; import jdk.jshell.spi.ExecutionEnv; +import jsheets.evaluation.sandbox.SandboxLoader; import jsheets.evaluation.sandbox.validation.Rule; import java.util.Collection; import java.util.Map; import java.util.Objects; -public final class SandboxedEnvironment implements ExecutionControlProvider { +public final class SandboxedEnvironment + implements ExecutionEnvironment, ExecutionControlProvider { + public static SandboxedEnvironment create(Collection rules) { Objects.requireNonNull(rules); - return new SandboxedEnvironment(rules); + return new SandboxedEnvironment(SandboxLoader.create(rules)); } - private final Collection rules; + private final SandboxLoader loader; + + private SandboxedEnvironment(SandboxLoader loader) { + this.loader = loader; + } - private SandboxedEnvironment(Collection rules) { - this.rules = rules; + @Override + public Installation install() { + return loader.install()::run; + } + + @Override + public ExecutionControlProvider control(String name) { + return this; } @Override @@ -32,6 +45,6 @@ public ExecutionControl generate( ExecutionEnv environment, Map parameters ) { - return new DirectExecutionControl(SandboxLoader.create(rules)); + return new DirectExecutionControl(loader); } } diff --git a/evaluation/src/main/java/jsheets/evaluation/shell/execution/ExhaustiveExecution.java b/evaluation/src/main/java/jsheets/evaluation/shell/execution/ExhaustiveExecution.java index 9b94b65..5ae62aa 100644 --- a/evaluation/src/main/java/jsheets/evaluation/shell/execution/ExhaustiveExecution.java +++ b/evaluation/src/main/java/jsheets/evaluation/shell/execution/ExhaustiveExecution.java @@ -267,7 +267,12 @@ static T run(ResolveTask task) { gutsAccessor = resolveGutsAccessor(); } catch (Throwable failure) { resolveError = failure.getMessage(); - log.atWarning().withCause(failure).log("failed to initialize"); + if (failure.getCause() instanceof IllegalAccessException) { + log.atWarning() + .log("ExhaustiveExecution is not possible: jdk.jshell is not open"); + } else { + log.atWarning().withCause(failure).log("failed to initialize"); + } } } } \ No newline at end of file diff --git a/evaluation/src/test/java/jsheets/evaluation/sandbox/SandboxExecutionTest.java b/evaluation/src/test/java/jsheets/evaluation/sandbox/SandboxExecutionTest.java index bed0abd..66564b3 100644 --- a/evaluation/src/test/java/jsheets/evaluation/sandbox/SandboxExecutionTest.java +++ b/evaluation/src/test/java/jsheets/evaluation/sandbox/SandboxExecutionTest.java @@ -4,6 +4,7 @@ import jsheets.evaluation.sandbox.access.AccessGraph; import jsheets.evaluation.sandbox.validation.Analysis; import jsheets.evaluation.sandbox.validation.ForbiddenMemberFilter; +import jsheets.evaluation.shell.environment.SandboxedEnvironment; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/evaluation/src/test/java/jsheets/evaluation/shell/ExhaustiveExecutionTest.java b/evaluation/src/test/java/jsheets/evaluation/shell/ExhaustiveExecutionTest.java index bc581b9..f5ce0f5 100644 --- a/evaluation/src/test/java/jsheets/evaluation/shell/ExhaustiveExecutionTest.java +++ b/evaluation/src/test/java/jsheets/evaluation/shell/ExhaustiveExecutionTest.java @@ -69,7 +69,6 @@ public void testEmpty() { public void testInvalid() { var events = ExhaustiveExecution.create(createSilentShell()) .execute("callWithoutClosedParen(someArgument, 1, 2, 3"); - System.out.println(events); Assertions.assertEquals(1, events.size()); } } \ No newline at end of file diff --git a/runtime/README.md b/runtime/README.md index cd801a0..c8cdcb4 100644 --- a/runtime/README.md +++ b/runtime/README.md @@ -11,6 +11,33 @@ allowing it to be scaled horizontally. This is beneficial, as user code evaluation poses a chance for *JVM crashes*. +### Running +This component should be run using the docker image: it requires some +special startup configuration to work and has dependencies that are not bundled +in the build *jar archive*. + +If you wish to run it manually, ensure that all required libraries are provided +(in the runtime classpath) and open the `jdk.jshell` module to all unnamed modules: +`--add-opens jdk.jshell/jdk.jshell=ALL-UNNAMED`, the latter is required to use +the *exhaustive execution* feature. + +For more information inspect the image's [run script](./deploy/entrypoint.sh). + +### Configuration + +**NOTE** That every environment variable is prefixed with `JSHEETS_RUNTIME_`, +thus `SERVER_PORT` has to be specified as `JSHEETS_RUNTIME_SERVER_PORT`. + +| Key | Environment Suffix | Default | Description | +|-----|-------------|---------|-------------| +| server.port | `SERVER_PORT` | `8080` | gRpc Server Port | +| server.feature.enableHealthService | `FEATURE_ENABLE_HEALTH_CHECK` | `true` | Toggles the Health Service | +| server.feature.enableGrpcReflection | `FEATURE_ENABLE_GRPC_REFLECTION` | `false` | Toggles the Health Service | +| +| service.id | `SERVICE_ID` | *generated* | Id that this service is advertised with | +| service.advertisedHost | `SERVICE_ADVERTISED_HOST` | none | The endpoint that is advertised in the service discovery | +| evaluation.sandbox.disable | `EVALUATION_SANDBOX_DISABLE` | `false` | Disables the sandbox for code execution **dangerous** | + ### Sandboxing The JVM itself is a sufficient sandbox, if we restrict the methods that can be called to that of classes without side effects to the system @@ -55,13 +82,13 @@ Given the following class `Library` in package `evilcorp.coolib` package evilcorp.coolib; class Library { - int count(int[] integers) { ... } + int count(int[] integers) { /*...*/ } - int count(double[] doubles) { ... } + int count(double[] doubles) { /*...*/ } - int count(float[] floats) { ... } + int count(float[] floats) { /*...*/ } - long count(int[][] twoDimensionIntegers) { ... } + long count(int[][] twoDimensionIntegers) { /*...*/ } void quit() { System.exit(-1); @@ -97,9 +124,6 @@ security risk (like `System#identityHashCode(Object)`), but also methods like `System#exit(int)`, we would thus be very careful with granting access to this class. - - - ### Scaling Since the *runtime* does not save any data and its state only consists of the active evaluations, it can be scaled horizontally to **thousands** of diff --git a/runtime/src/main/java/jsheets/runtime/ConfigModule.java b/runtime/src/main/java/jsheets/runtime/ConfigModule.java index b117569..716a430 100644 --- a/runtime/src/main/java/jsheets/runtime/ConfigModule.java +++ b/runtime/src/main/java/jsheets/runtime/ConfigModule.java @@ -21,7 +21,7 @@ static ConfigModule create() { private ConfigModule() {} - private static final String environmentPrefix = "JSHELL_RUNTIME"; + private static final String environmentPrefix = "JSHEETS_RUNTIME"; @Provides @Singleton diff --git a/runtime/src/main/java/jsheets/runtime/ServerSetupModule.java b/runtime/src/main/java/jsheets/runtime/ServerSetupModule.java index c6db93e..1979695 100644 --- a/runtime/src/main/java/jsheets/runtime/ServerSetupModule.java +++ b/runtime/src/main/java/jsheets/runtime/ServerSetupModule.java @@ -1,11 +1,6 @@ package jsheets.runtime; -import java.util.Collection; -import java.util.EnumSet; -import java.util.List; -import java.util.Set; -import java.util.UUID; -import java.util.function.Supplier; +import java.util.*; import com.google.common.net.HostAndPort; @@ -14,6 +9,7 @@ import io.grpc.ServerServiceDefinition; import javax.inject.Named; +import javax.inject.Provider; import javax.inject.Singleton; import jsheets.config.Config; import jsheets.runtime.discovery.AdvertisementHook; @@ -83,12 +79,12 @@ String serviceId(Config config) { @Singleton Collection setupHooks( Config config, - Supplier advertisementChannelFactory + Provider> advertisementChannelFactory ) { return advertisedHostKey.in(config).orNone().map(host -> { var hook = AdvertisementHook.create( host, - advertisementChannelFactory.get() + advertisementChannelFactory.get().orElseThrow() ); return List.of(hook); }).orElse(List.of()); diff --git a/runtime/src/main/java/jsheets/runtime/evaluation/EvaluationModule.java b/runtime/src/main/java/jsheets/runtime/evaluation/EvaluationModule.java index 256b92d..1a13f80 100644 --- a/runtime/src/main/java/jsheets/runtime/evaluation/EvaluationModule.java +++ b/runtime/src/main/java/jsheets/runtime/evaluation/EvaluationModule.java @@ -6,10 +6,20 @@ import javax.inject.Named; import jsheets.evaluation.EvaluationEngine; +import jsheets.evaluation.sandbox.access.AccessGraph; +import jsheets.evaluation.sandbox.validation.ForbiddenMemberFilter; import jsheets.evaluation.shell.ShellEvaluationEngine; +import jsheets.evaluation.shell.environment.ExecutionEnvironment; +import jsheets.evaluation.shell.environment.SandboxedEnvironment; +import jsheets.evaluation.shell.environment.StandardEnvironment; +import jsheets.evaluation.shell.environment.inprocess.EmbeddedEnvironment; +import jsheets.evaluation.shell.environment.inprocess.InProcessExecutionControl; import jsheets.evaluation.shell.execution.ExecutionMethod; import jsheets.evaluation.shell.execution.ExhaustiveExecution; import jsheets.config.Config; +import jsheets.evaluation.shell.execution.SystemBasedExecutionMethodFactory; + +import java.util.List; import static jsheets.runtime.evaluation.SandboxConfigSource.accessGraphKey; import static jsheets.runtime.evaluation.SandboxConfigSource.disableSandboxKey; @@ -23,30 +33,25 @@ private EvaluationModule() {} @Provides @Singleton - EvaluationEngine evaluationEngine(ExecutionMethod.Factory executionMethodFactory) { + EvaluationEngine evaluationEngine(ExecutionEnvironment environment) { return ShellEvaluationEngine.newBuilder() - .useExecutionMethodFactory(executionMethodFactory) + .useEnvironment(environment) + .useExecutionMethodFactory(SystemBasedExecutionMethodFactory.create()) .create(); } @Provides @Singleton - ExecutionMethod.Factory executionMethodFactory( - Config config, - @Named("underlyingExecutionMethod") - ExecutionMethod.Factory underlyingExecutionMethodFactory - ) { + ExecutionEnvironment executionEnvironment(Config config) { boolean disableSandbox = disableSandboxKey().in(config).orNone().orElse(false); - var accessGraphConfig = accessGraphKey().in(config).require(); if (disableSandbox) { - return underlyingExecutionMethodFactory; + return StandardEnvironment.create(); } - return underlyingExecutionMethodFactory; - } - - @Named("underlyingExecutionMethod") - ExecutionMethod.Factory underlyingExecutionMethodFactory(Config config) { - return ExhaustiveExecution::create; + var accessGraphConfig = accessGraphKey().in(config).require(); + var accessGraph = AccessGraph.of(accessGraphConfig.split("\n")); + return SandboxedEnvironment.create( + List.of(ForbiddenMemberFilter.create(accessGraph)) + ); } } \ No newline at end of file diff --git a/runtime/src/main/resources/runtime/evaluation/defaultImports.txt b/runtime/src/main/resources/runtime/evaluation/defaultImports.txt new file mode 100644 index 0000000..75bfda6 --- /dev/null +++ b/runtime/src/main/resources/runtime/evaluation/defaultImports.txt @@ -0,0 +1,7 @@ +java.lang.* +java.math.* +java.time.* +java.text.* +java.util.* +java.util.function.* +java.util.stream.* \ No newline at end of file diff --git a/runtime/src/main/resources/runtime/evaluation/sandbox/accessGraph.txt b/runtime/src/main/resources/runtime/evaluation/sandbox/accessGraph.txt index e69de29..aaea75b 100644 --- a/runtime/src/main/resources/runtime/evaluation/sandbox/accessGraph.txt +++ b/runtime/src/main/resources/runtime/evaluation/sandbox/accessGraph.txt @@ -0,0 +1,2 @@ +java.lang.Object +java.lang.Integer \ No newline at end of file From 0948575ca26375af039f8eff708d20914f28b440 Mon Sep 17 00:00:00 2001 From: Merlin Osayimwen Date: Fri, 15 Oct 2021 18:32:43 +0200 Subject: [PATCH 10/14] Add runtime run documentation --- runtime/README.md | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/runtime/README.md b/runtime/README.md index c8cdcb4..3b5c06f 100644 --- a/runtime/README.md +++ b/runtime/README.md @@ -12,10 +12,31 @@ horizontally. This is beneficial, as user code evaluation poses a chance for *JVM crashes*. ### Running -This component should be run using the docker image: it requires some -special startup configuration to work and has dependencies that are not bundled -in the build *jar archive*. +This component should be run using the docker image, since it requires +special configuration for all features to work and does now bundle its +dependencies in the build *jar archive*. +#### Docker +The image tag is `ehenoma/jsheets-runtime:latest` and is located in the +[deploy](./deploy) folder. + +Following [Docker Compose](https://docs.docker.com/compose/) file is sufficient +to run a configured instance of the runtime. + +```yml +version: "3.7" +services: + runtime: + image: ehenoma/jsheets-runtime:latest + container_name: jsheets-runtime + hostname: jsheets-runtime + environment: + JSHEETS_RUNTIME_SERVER_FEATURES_ENABLE_GRPC_REFLECTION: "true" + JSHEETS_RUNTIME_SERVER_SERVICE_ID: "my-only-service" + ports: + - "8080:8080" +``` +#### Manual If you wish to run it manually, ensure that all required libraries are provided (in the runtime classpath) and open the `jdk.jshell` module to all unnamed modules: `--add-opens jdk.jshell/jdk.jshell=ALL-UNNAMED`, the latter is required to use @@ -31,8 +52,8 @@ thus `SERVER_PORT` has to be specified as `JSHEETS_RUNTIME_SERVER_PORT`. | Key | Environment Suffix | Default | Description | |-----|-------------|---------|-------------| | server.port | `SERVER_PORT` | `8080` | gRpc Server Port | -| server.feature.enableHealthService | `FEATURE_ENABLE_HEALTH_CHECK` | `true` | Toggles the Health Service | -| server.feature.enableGrpcReflection | `FEATURE_ENABLE_GRPC_REFLECTION` | `false` | Toggles the Health Service | +| server.features.enableHealthService | `FEATURES_ENABLE_HEALTH_CHECK` | `true` | Toggles the Health Service | +| server.features.enableGrpcReflection | `FEATURES_ENABLE_GRPC_REFLECTION` | `false` | Toggles the Health Service | | | service.id | `SERVICE_ID` | *generated* | Id that this service is advertised with | | service.advertisedHost | `SERVICE_ADVERTISED_HOST` | none | The endpoint that is advertised in the service discovery | From 3d8d9908416367ab8a0353b2bc3c31c78742a0ed Mon Sep 17 00:00:00 2001 From: Merlin Osayimwen Date: Fri, 15 Oct 2021 18:34:54 +0200 Subject: [PATCH 11/14] Fix build errors --- .../jsheets/evaluation/shell/ShellEvaluationEngineTest.java | 2 -- .../java/jsheets/server/evaluation/EvaluationModule.java | 5 +---- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/evaluation/src/test/java/jsheets/evaluation/shell/ShellEvaluationEngineTest.java b/evaluation/src/test/java/jsheets/evaluation/shell/ShellEvaluationEngineTest.java index 7d55d8b..c26d359 100644 --- a/evaluation/src/test/java/jsheets/evaluation/shell/ShellEvaluationEngineTest.java +++ b/evaluation/src/test/java/jsheets/evaluation/shell/ShellEvaluationEngineTest.java @@ -1,6 +1,5 @@ package jsheets.evaluation.shell; -import java.time.Clock; import java.util.UUID; import jsheets.EvaluateResponse; @@ -27,7 +26,6 @@ public void testExecution() { var environment = StandardEnvironment.create(); var installation = environment.install(); var engine = ShellEvaluationEngine.newBuilder() - .useClock(Clock.systemUTC()) .useEnvironment(environment) .useWorkerPool(Runnable::run) .create(); diff --git a/server/src/main/java/jsheets/server/evaluation/EvaluationModule.java b/server/src/main/java/jsheets/server/evaluation/EvaluationModule.java index 1ada105..9a601b0 100644 --- a/server/src/main/java/jsheets/server/evaluation/EvaluationModule.java +++ b/server/src/main/java/jsheets/server/evaluation/EvaluationModule.java @@ -1,6 +1,5 @@ package jsheets.server.evaluation; -import java.time.Clock; import java.util.concurrent.Executors; import com.google.inject.AbstractModule; @@ -10,7 +9,6 @@ import jsheets.evaluation.EvaluationEngine; import jsheets.evaluation.shell.ShellEvaluationEngine; - public final class EvaluationModule extends AbstractModule { public static EvaluationModule create() { return new EvaluationModule(); @@ -20,10 +18,9 @@ private EvaluationModule() {} @Provides @Singleton - EvaluationEngine evaluationEngine(Clock clock) { + EvaluationEngine evaluationEngine() { return ShellEvaluationEngine.newBuilder() .useWorkerPool(Executors.newCachedThreadPool()) - .useClock(clock) .create(); } } \ No newline at end of file From 5f9f1ac7e8e657aed16032b6ee308339f5df51bf Mon Sep 17 00:00:00 2001 From: Merlin Osayimwen Date: Fri, 15 Oct 2021 22:41:14 +0200 Subject: [PATCH 12/14] Connect runtime with server --- build.gradle | 1 + deploy/minimal/docker-compose.yml | 54 ++++++++++-- docker-compose.build.yml | 13 +++ docker-compose.dev.yml | 27 ------ .../main/java/jsheets/config/CamelCase.java | 0 .../java/jsheets/config/CombinedConfig.java | 0 .../src/main/java/jsheets/config/Config.java | 0 .../jsheets/config/EnvironmentConfig.java | 0 .../java/jsheets/config/MissingField.java | 0 .../main/java/jsheets/config/RawConfig.java | 0 .../java/jsheets/config/ResolvedField.java | 0 .../evaluation/sandbox/SandboxConfig.java | 7 -- .../environment/SandboxedEnvironment.java | 11 +-- .../java/jsheets/config/CamelCaseTest.java | 0 .../jsheets/config/EnvironmentConfigTest.java | 12 ++- runtime/README.md | 2 + runtime/build.gradle | 5 +- runtime/deploy/Dockerfile | 2 +- .../config/consul/ConsulConfigSource.java | 50 ----------- .../src/main/java/jsheets/runtime/App.java | 2 +- .../java/jsheets/runtime/ConfigModule.java | 8 +- .../java/jsheets/runtime/ConsulModule.java | 58 ------------- .../jsheets/runtime/ServerSetupModule.java | 2 + .../runtime/SnippetRuntimeService.java | 13 ++- .../java/jsheets/runtime/ZookeeperModule.java | 56 +++++++++++++ .../runtime/discovery/AdvertisementHook.java | 23 +++++- .../ConsulServiceAdvertisementChannel.java | 82 ------------------- .../ServiceAdvertisementChannel.java | 4 +- .../ZookeeperServiceAdvertisementChannel.java | 81 ++++++++++++++++++ .../runtime/evaluation/EvaluationModule.java | 5 -- .../evaluation/sandbox/accessGraph.txt | 15 +++- server/build.gradle | 3 + server/src/main/java/jsheets/server/App.java | 5 +- .../java/jsheets/server/ConfigModule.java | 21 +++++ .../src/main/java/jsheets/server/Server.java | 10 +-- .../java/jsheets/server/ServerModule.java | 4 +- .../evaluation/EvaluationConnection.java | 26 +++++- .../server/evaluation/EvaluationModule.java | 77 ++++++++++++++++- .../client/ConsulServiceDiscovery.java | 19 ----- .../client/PooledEvaluationEngine.java | 34 ++++++++ .../client/SnippetRuntimeEngine.java | 2 +- .../client/ZookeeperEngineDiscovery.java | 43 ++++++++++ website/src/client/index.ts | 1 + 43 files changed, 480 insertions(+), 298 deletions(-) create mode 100644 docker-compose.build.yml delete mode 100644 docker-compose.dev.yml rename {runtime => evaluation}/src/main/java/jsheets/config/CamelCase.java (100%) rename {runtime => evaluation}/src/main/java/jsheets/config/CombinedConfig.java (100%) rename {runtime => evaluation}/src/main/java/jsheets/config/Config.java (100%) rename {runtime => evaluation}/src/main/java/jsheets/config/EnvironmentConfig.java (100%) rename {runtime => evaluation}/src/main/java/jsheets/config/MissingField.java (100%) rename {runtime => evaluation}/src/main/java/jsheets/config/RawConfig.java (100%) rename {runtime => evaluation}/src/main/java/jsheets/config/ResolvedField.java (100%) delete mode 100644 evaluation/src/main/java/jsheets/evaluation/sandbox/SandboxConfig.java rename {runtime => evaluation}/src/test/java/jsheets/config/CamelCaseTest.java (100%) rename {runtime => evaluation}/src/test/java/jsheets/config/EnvironmentConfigTest.java (91%) delete mode 100644 runtime/src/main/java/jsheets/config/consul/ConsulConfigSource.java delete mode 100644 runtime/src/main/java/jsheets/runtime/ConsulModule.java create mode 100644 runtime/src/main/java/jsheets/runtime/ZookeeperModule.java delete mode 100644 runtime/src/main/java/jsheets/runtime/discovery/ConsulServiceAdvertisementChannel.java create mode 100644 runtime/src/main/java/jsheets/runtime/discovery/ZookeeperServiceAdvertisementChannel.java create mode 100644 server/src/main/java/jsheets/server/ConfigModule.java delete mode 100644 server/src/main/java/jsheets/server/evaluation/client/ConsulServiceDiscovery.java create mode 100644 server/src/main/java/jsheets/server/evaluation/client/PooledEvaluationEngine.java create mode 100644 server/src/main/java/jsheets/server/evaluation/client/ZookeeperEngineDiscovery.java diff --git a/build.gradle b/build.gradle index 0248c5e..a1cd562 100644 --- a/build.gradle +++ b/build.gradle @@ -5,6 +5,7 @@ allprojects { } ext { + curatorVersion = '5.2.0' grpcVersion = '1.41.0' floggerVersion = '0.7.1' junitPlatformVersion = '5.8.1' diff --git a/deploy/minimal/docker-compose.yml b/deploy/minimal/docker-compose.yml index 647a44a..abd972a 100644 --- a/deploy/minimal/docker-compose.yml +++ b/deploy/minimal/docker-compose.yml @@ -2,21 +2,65 @@ version: '3.7' services: server: image: ehenoma/jsheets:latest + build: + context: ../../ + dockerfile: ./server/deploy/Dockerfile + restart: always environment: JSHEETS_SERVER_PORT: 8080 JSHEETS_MONGODB_URI: mongodb://root:root@document-store/jsheets + JSHEETS_ZOOKEEPER_CONNECTION_STRING: zookeeper:2181 ports: - "8080:8080" networks: - - database + - document-store + - zookeeper + - runtime document-store: - container_name: document-store image: mongo:latest + container_name: document-store + restart: always environment: MONGO_INITDB_ROOT_USERNAME: root MONGO_INITDB_ROOT_PASSWORD: root networks: - - database - + - document-store + zookeeper: + image: zookeeper + restart: always + hostname: zookeeper + ports: + - "2181:2181" + environment: + ZOO_MY_ID: 1 + ZOO_PORT: 2181 + networks: + - zookeeper + zookeeper-web: + image: elkozmon/zoonavigator:latest + restart: unless-stopped + environment: + HTTP_PORT: 9000 + ports: + - "9000:9000" + networks: + - zookeeper + runtime: + image: ehenoma/jsheets-runtime:latest + build: + context: ../../ + dockerfile: ./runtime/deploy/Dockerfile + restart: always + container_name: runtime + hostname: runtime + environment: + JSHEETS_RUNTIME_SERVER_PORT: 8080 + JSHEETS_RUNTIME_ZOOKEEPER_CONNECTION_STRING: zookeeper:2181 + JSHEETS_RUNTIME_SERVICE_ADVERTISED_HOST: runtime:8080 + networks: + - zookeeper + - runtime networks: - database: \ No newline at end of file + document-store: + zookeeper: + runtime: \ No newline at end of file diff --git a/docker-compose.build.yml b/docker-compose.build.yml new file mode 100644 index 0000000..765a511 --- /dev/null +++ b/docker-compose.build.yml @@ -0,0 +1,13 @@ +# Make sure to run 'gradle build copyDependencies' before building this +version: '3.7' +services: + server: + image: ehenoma/jsheets:latest + build: + dockerfile: server/deploy/Dockerfile + context: . + runtime: + image: ehenoma/jsheets-latest:latest + build: + dockerfile: runtime/deploy/Dockerfile + context: . \ No newline at end of file diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml deleted file mode 100644 index c72c7e3..0000000 --- a/docker-compose.dev.yml +++ /dev/null @@ -1,27 +0,0 @@ -version: '3.7' -services: - server: - image: ehenoma/jsheets:dev - build: - # Make sure to run 'gradle copyDependencies' before building - dockerfile: server/deploy/Dockerfile - context: . - container_name: jsheets-server - environment: - JSHEETS_SERVER_PORT: 8080 - JSHEETS_MONGODB_URI: mongodb://root:root@document-store/jsheets - ports: - - "8080:8080" - networks: - - database - document-store: - container_name: document-store - image: mongo:latest - environment: - MONGO_INITDB_ROOT_USERNAME: root - MONGO_INITDB_ROOT_PASSWORD: root - networks: - - database - -networks: - database: \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/config/CamelCase.java b/evaluation/src/main/java/jsheets/config/CamelCase.java similarity index 100% rename from runtime/src/main/java/jsheets/config/CamelCase.java rename to evaluation/src/main/java/jsheets/config/CamelCase.java diff --git a/runtime/src/main/java/jsheets/config/CombinedConfig.java b/evaluation/src/main/java/jsheets/config/CombinedConfig.java similarity index 100% rename from runtime/src/main/java/jsheets/config/CombinedConfig.java rename to evaluation/src/main/java/jsheets/config/CombinedConfig.java diff --git a/runtime/src/main/java/jsheets/config/Config.java b/evaluation/src/main/java/jsheets/config/Config.java similarity index 100% rename from runtime/src/main/java/jsheets/config/Config.java rename to evaluation/src/main/java/jsheets/config/Config.java diff --git a/runtime/src/main/java/jsheets/config/EnvironmentConfig.java b/evaluation/src/main/java/jsheets/config/EnvironmentConfig.java similarity index 100% rename from runtime/src/main/java/jsheets/config/EnvironmentConfig.java rename to evaluation/src/main/java/jsheets/config/EnvironmentConfig.java diff --git a/runtime/src/main/java/jsheets/config/MissingField.java b/evaluation/src/main/java/jsheets/config/MissingField.java similarity index 100% rename from runtime/src/main/java/jsheets/config/MissingField.java rename to evaluation/src/main/java/jsheets/config/MissingField.java diff --git a/runtime/src/main/java/jsheets/config/RawConfig.java b/evaluation/src/main/java/jsheets/config/RawConfig.java similarity index 100% rename from runtime/src/main/java/jsheets/config/RawConfig.java rename to evaluation/src/main/java/jsheets/config/RawConfig.java diff --git a/runtime/src/main/java/jsheets/config/ResolvedField.java b/evaluation/src/main/java/jsheets/config/ResolvedField.java similarity index 100% rename from runtime/src/main/java/jsheets/config/ResolvedField.java rename to evaluation/src/main/java/jsheets/config/ResolvedField.java diff --git a/evaluation/src/main/java/jsheets/evaluation/sandbox/SandboxConfig.java b/evaluation/src/main/java/jsheets/evaluation/sandbox/SandboxConfig.java deleted file mode 100644 index 916bd20..0000000 --- a/evaluation/src/main/java/jsheets/evaluation/sandbox/SandboxConfig.java +++ /dev/null @@ -1,7 +0,0 @@ -package jsheets.evaluation.sandbox; - -public final class SandboxConfig { - private SandboxConfig() {} - - -} \ No newline at end of file diff --git a/evaluation/src/main/java/jsheets/evaluation/shell/environment/SandboxedEnvironment.java b/evaluation/src/main/java/jsheets/evaluation/shell/environment/SandboxedEnvironment.java index 2abd2d1..b308b48 100644 --- a/evaluation/src/main/java/jsheets/evaluation/shell/environment/SandboxedEnvironment.java +++ b/evaluation/src/main/java/jsheets/evaluation/shell/environment/SandboxedEnvironment.java @@ -10,24 +10,25 @@ import java.util.Collection; import java.util.Map; import java.util.Objects; +import java.util.function.Supplier; public final class SandboxedEnvironment implements ExecutionEnvironment, ExecutionControlProvider { public static SandboxedEnvironment create(Collection rules) { Objects.requireNonNull(rules); - return new SandboxedEnvironment(SandboxLoader.create(rules)); + return new SandboxedEnvironment(() -> SandboxLoader.create(rules)); } - private final SandboxLoader loader; + private final Supplier loader; - private SandboxedEnvironment(SandboxLoader loader) { + private SandboxedEnvironment(Supplier loader) { this.loader = loader; } @Override public Installation install() { - return loader.install()::run; + return loader.get().install()::run; } @Override @@ -45,6 +46,6 @@ public ExecutionControl generate( ExecutionEnv environment, Map parameters ) { - return new DirectExecutionControl(loader); + return new DirectExecutionControl(loader.get()); } } diff --git a/runtime/src/test/java/jsheets/config/CamelCaseTest.java b/evaluation/src/test/java/jsheets/config/CamelCaseTest.java similarity index 100% rename from runtime/src/test/java/jsheets/config/CamelCaseTest.java rename to evaluation/src/test/java/jsheets/config/CamelCaseTest.java diff --git a/runtime/src/test/java/jsheets/config/EnvironmentConfigTest.java b/evaluation/src/test/java/jsheets/config/EnvironmentConfigTest.java similarity index 91% rename from runtime/src/test/java/jsheets/config/EnvironmentConfigTest.java rename to evaluation/src/test/java/jsheets/config/EnvironmentConfigTest.java index ea772f6..a8c8bb7 100644 --- a/runtime/src/test/java/jsheets/config/EnvironmentConfigTest.java +++ b/evaluation/src/test/java/jsheets/config/EnvironmentConfigTest.java @@ -5,8 +5,6 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; - public class EnvironmentConfigTest { @Test public void testLookup() { @@ -40,23 +38,23 @@ public void testLookup() { @Test public void testTranslateKey() { - assertEquals( + Assertions.assertEquals( "OPINION_THE_BEST_SONG", EnvironmentConfig.translateKey("opinion.theBestSong") ); - assertEquals( + Assertions.assertEquals( "ARTIST_NINA_SIMONE_BLUES", EnvironmentConfig.translateKey("artist.nina_simone.blues") ); - assertEquals( + Assertions.assertEquals( "PLAYBACK_SONG_NAME", EnvironmentConfig.translateKey("playback.song.name") ); - assertEquals( + Assertions.assertEquals( "PLAYBACK_SONG_NAME", EnvironmentConfig.translateKey("playback.songName") ); - assertEquals( + Assertions.assertEquals( "PLAYBACK_SONG_NAME", EnvironmentConfig.translateKey("playbackSongName") ); diff --git a/runtime/README.md b/runtime/README.md index 3b5c06f..f6f4faa 100644 --- a/runtime/README.md +++ b/runtime/README.md @@ -58,6 +58,8 @@ thus `SERVER_PORT` has to be specified as `JSHEETS_RUNTIME_SERVER_PORT`. | service.id | `SERVICE_ID` | *generated* | Id that this service is advertised with | | service.advertisedHost | `SERVICE_ADVERTISED_HOST` | none | The endpoint that is advertised in the service discovery | | evaluation.sandbox.disable | `EVALUATION_SANDBOX_DISABLE` | `false` | Disables the sandbox for code execution **dangerous** | +| zookeeper.connectionString | `ZOOKEEPER_CONNECTION_STRING` | none | Connection string to zookeeper | +| zookeeper.connectBackoff | `ZOOKEEPER_CONNECT_BACKOFF` | `1000` | Initial backoff after failed zookeeper connection | ### Sandboxing The JVM itself is a sufficient sandbox, if we restrict the methods diff --git a/runtime/build.gradle b/runtime/build.gradle index 644fa29..b9b1603 100644 --- a/runtime/build.gradle +++ b/runtime/build.gradle @@ -25,7 +25,7 @@ ext { dependencies { implementation project(':protocol') implementation project(':evaluation') - implementation "com.ecwid.consul:consul-api:$consuleClientVersion" + implementation "org.apache.curator:curator-x-discovery:$curatorVersion" implementation "io.micrometer:micrometer-core:$micrometerVersion" implementation "org.mongodb:mongodb-driver-sync:$mongoDbDriverVersion" implementation "org.slf4j:slf4j-simple:$slf4jVersion" @@ -62,6 +62,7 @@ processResources { } jar { + archiveName 'app.jar' duplicatesStrategy = DuplicatesStrategy.INCLUDE manifest { attributes( @@ -78,7 +79,7 @@ def dockerImage = "$dockerImageName:$dockerImageTag" task buildDocker(type: Exec) { dependsOn copyDependencies, build workingDir "$projectDir" - commandLine "docker", "build", "--rm", ".", "-t", dockerImage, "-f", "./server/deploy/Dockerfile" + commandLine "docker", "build", "--rm", ".", "-t", dockerImage, "-f", "./runtime/deploy/Dockerfile" } task justRunDocker(type: Exec) { diff --git a/runtime/deploy/Dockerfile b/runtime/deploy/Dockerfile index aff840f..7b51c07 100644 --- a/runtime/deploy/Dockerfile +++ b/runtime/deploy/Dockerfile @@ -38,7 +38,7 @@ WORKDIR /app COPY --from=link-java /usr/src/build/jre jre COPY --from=link-java /usr/src/build/libs/* libs/ COPY --from=link-java /usr/src/build/app.jar app.jar -ADD /server/deploy/entrypoint.sh entrypoint.sh +ADD /runtime/deploy/entrypoint.sh entrypoint.sh ADD /website/build static/ RUN chmod +x entrypoint.sh diff --git a/runtime/src/main/java/jsheets/config/consul/ConsulConfigSource.java b/runtime/src/main/java/jsheets/config/consul/ConsulConfigSource.java deleted file mode 100644 index f19dbcb..0000000 --- a/runtime/src/main/java/jsheets/config/consul/ConsulConfigSource.java +++ /dev/null @@ -1,50 +0,0 @@ -package jsheets.config.consul; - -import java.util.Objects; - -import com.ecwid.consul.v1.ConsulClient; - -import jsheets.config.Config; -import jsheets.config.RawConfig; - -/** - * Loads configuration from a Consul backend. - */ -public final class ConsulConfigSource implements Config.Source { - public static ConsulConfigSource prefixed(String prefix, ConsulClient client) { - Objects.requireNonNull(client, "client"); - Objects.requireNonNull(prefix, "prefix"); - return new ConsulConfigSource(prefix + ".", client); - } - - private final String prefix; - private final ConsulClient client; - - private ConsulConfigSource(String prefix, ConsulClient client) { - this.prefix = prefix; - this.client = client; - } - - @Override - public Config load() { - var pairs = client.getKVValues(prefix).getValue(); - var config = RawConfig.newBuilder(); - for (var pair : pairs) { - var key = removePrefix(pair.getKey()); - config.withRaw(key, pair.getDecodedValue()); - } - return config.create(); - } - - private String removePrefix(String key) { - return key.startsWith(prefix) - ? key.substring(prefix.length()) - : key; - } - - @Override - public String toString() { - return "ConsulConfigSource(prefix=%s, client=%s)" - .formatted(prefix, client); - } -} \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/runtime/App.java b/runtime/src/main/java/jsheets/runtime/App.java index d9a6a60..04c3e5e 100644 --- a/runtime/src/main/java/jsheets/runtime/App.java +++ b/runtime/src/main/java/jsheets/runtime/App.java @@ -39,7 +39,7 @@ private static Injector configureInjector() { return Guice.createInjector( ServerSetupModule.create(), ConfigModule.create(), - ConsulModule.create(), + ZookeeperModule.create(), EvaluationModule.create() ); } diff --git a/runtime/src/main/java/jsheets/runtime/ConfigModule.java b/runtime/src/main/java/jsheets/runtime/ConfigModule.java index 716a430..dab7fb6 100644 --- a/runtime/src/main/java/jsheets/runtime/ConfigModule.java +++ b/runtime/src/main/java/jsheets/runtime/ConfigModule.java @@ -1,7 +1,6 @@ package jsheets.runtime; import java.util.ArrayList; -import java.util.Optional; import com.google.inject.AbstractModule; import com.google.inject.Provides; @@ -11,7 +10,6 @@ import jsheets.config.CombinedConfig; import jsheets.config.Config; import jsheets.config.EnvironmentConfig; -import jsheets.config.consul.ConsulConfigSource; import jsheets.runtime.evaluation.SandboxConfigSource; final class ConfigModule extends AbstractModule { @@ -25,14 +23,10 @@ private ConfigModule() {} @Provides @Singleton - Config createConfig( - Optional consulSource, - @Named("environment") Config environment - ) { + Config createConfig(@Named("environment") Config environment) { var configs = new ArrayList(); configs.add(environment); configs.add(SandboxConfigSource.fromClassPath().load()); - consulSource.ifPresent(source -> configs.add(source.load())); return CombinedConfig.of(configs.toArray(Config[]::new)); } diff --git a/runtime/src/main/java/jsheets/runtime/ConsulModule.java b/runtime/src/main/java/jsheets/runtime/ConsulModule.java deleted file mode 100644 index 804ff63..0000000 --- a/runtime/src/main/java/jsheets/runtime/ConsulModule.java +++ /dev/null @@ -1,58 +0,0 @@ -package jsheets.runtime; - -import java.util.Optional; - -import com.google.common.net.HostAndPort; - -import com.google.inject.AbstractModule; -import com.google.inject.Provides; -import com.google.inject.Singleton; - -import com.ecwid.consul.v1.ConsulClient; -import javax.inject.Named; -import jsheets.config.Config; -import jsheets.config.consul.ConsulConfigSource; -import jsheets.runtime.discovery.ConsulServiceAdvertisementChannel; -import jsheets.runtime.discovery.ServiceAdvertisementChannel; - -final class ConsulModule extends AbstractModule { - static ConsulModule create() { - return new ConsulModule(); - } - - private ConsulModule() {} - - private static final Config.Key consulEndpointKey = - Config.Key.of("consul.endpoint", HostAndPort::fromString); - - @Provides - @Singleton - Optional consulClient(@Named("environment") Config config) { - return consulEndpointKey.in(config).orNone().map(endpoint -> - new ConsulClient(endpoint.getHost(), endpoint.getPort()) - ); - } - - private static final String consulKeyPrefix = "jsheets.runtime"; - - @Provides - @Singleton - Optional consulConfigSource( - Optional clientBinding - ) { - return clientBinding.map(client -> - ConsulConfigSource.prefixed(consulKeyPrefix, client) - ); - } - - @Provides - @Singleton - Optional consulAdvertisementChannel( - @Named("serviceId") String serviceId, - Optional clientBinding - ) { - return clientBinding.map(client -> - ConsulServiceAdvertisementChannel.forServiceId(serviceId, client) - ); - } -} \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/runtime/ServerSetupModule.java b/runtime/src/main/java/jsheets/runtime/ServerSetupModule.java index 1979695..c948cc4 100644 --- a/runtime/src/main/java/jsheets/runtime/ServerSetupModule.java +++ b/runtime/src/main/java/jsheets/runtime/ServerSetupModule.java @@ -79,10 +79,12 @@ String serviceId(Config config) { @Singleton Collection setupHooks( Config config, + @Named("serviceId") String serviceId, Provider> advertisementChannelFactory ) { return advertisedHostKey.in(config).orNone().map(host -> { var hook = AdvertisementHook.create( + serviceId, host, advertisementChannelFactory.get().orElseThrow() ); diff --git a/runtime/src/main/java/jsheets/runtime/SnippetRuntimeService.java b/runtime/src/main/java/jsheets/runtime/SnippetRuntimeService.java index d60b3db..a8aaf89 100644 --- a/runtime/src/main/java/jsheets/runtime/SnippetRuntimeService.java +++ b/runtime/src/main/java/jsheets/runtime/SnippetRuntimeService.java @@ -69,6 +69,7 @@ private void processStop(StopEvaluationRequest request) { return; } evaluation.stop(); + evaluation = null; log.atFine().log("closed evaluation"); } @@ -92,7 +93,17 @@ private void processUnknown(EvaluateRequest request) { public void onError(Throwable failure) {} @Override - public void onCompleted() {} + public void onCompleted() { + lock.lock(); + try { + if (evaluation != null) { + evaluation.stop(); + evaluation = null; + } + } finally { + lock.unlock(); + } + } // Listens to EvaluationEngine @Override diff --git a/runtime/src/main/java/jsheets/runtime/ZookeeperModule.java b/runtime/src/main/java/jsheets/runtime/ZookeeperModule.java new file mode 100644 index 0000000..c371791 --- /dev/null +++ b/runtime/src/main/java/jsheets/runtime/ZookeeperModule.java @@ -0,0 +1,56 @@ +package jsheets.runtime; + +import java.util.Optional; + +import com.google.inject.AbstractModule; +import com.google.inject.Provides; + +import com.google.inject.Singleton; +import jsheets.config.Config; +import org.apache.curator.framework.CuratorFramework; + +import jsheets.runtime.discovery.ServiceAdvertisementChannel; +import jsheets.runtime.discovery.ZookeeperServiceAdvertisementChannel; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.ExponentialBackoffRetry; + +public class ZookeeperModule extends AbstractModule { + public static ZookeeperModule create() { + return new ZookeeperModule(); + } + + private ZookeeperModule() {} + + private static final Config.Key connectionStringKey + = Config.Key.ofString("zookeeper.connectionString"); + + private static final Config.Key connectionBackoffKey + = Config.Key.ofInt("zookeeper.connectBackoff"); + + private static final int defaultBackoff = 1000; + private static final int retryLimit = 3; + + @Provides + @Singleton + Optional curatorFramework(Config config) { + return connectionStringKey.in(config).orNone() + .map(connectionString -> { + int backoff = connectionBackoffKey.in(config).or(defaultBackoff); + var client = CuratorFrameworkFactory.newClient( + connectionString, + new ExponentialBackoffRetry(backoff, retryLimit) + ); + client.start(); + Runtime.getRuntime().addShutdownHook(new Thread(client::close)); + return client; + }); + } + + @Provides + @Singleton + Optional serviceAdvertisementChannel( + Optional curator + ) { + return curator.map(ZookeeperServiceAdvertisementChannel::create); + } +} diff --git a/runtime/src/main/java/jsheets/runtime/discovery/AdvertisementHook.java b/runtime/src/main/java/jsheets/runtime/discovery/AdvertisementHook.java index b968754..b148938 100644 --- a/runtime/src/main/java/jsheets/runtime/discovery/AdvertisementHook.java +++ b/runtime/src/main/java/jsheets/runtime/discovery/AdvertisementHook.java @@ -2,37 +2,52 @@ import java.util.Objects; +import com.google.common.flogger.FluentLogger; import com.google.common.net.HostAndPort; import javax.inject.Inject; +import javax.inject.Named; + import jsheets.runtime.ServerSetup; public final class AdvertisementHook implements ServerSetup.Hook { + private static final FluentLogger log = FluentLogger.forEnclosingClass(); + public static AdvertisementHook create( + String serviceId, HostAndPort advertisedHost, ServiceAdvertisementChannel channel ) { Objects.requireNonNull(advertisedHost, "advertisedHost"); Objects.requireNonNull(channel, "channel"); - return new AdvertisementHook(advertisedHost, channel); + return new AdvertisementHook(serviceId, advertisedHost, channel); } private final HostAndPort advertisedHost; private final ServiceAdvertisementChannel advertisementChannel; + private final String serviceId; private volatile ServiceAdvertisement advertisement; @Inject AdvertisementHook( + @Named("serviceId") String serviceId, HostAndPort advertisedHost, ServiceAdvertisementChannel advertisementChannel ) { + this.serviceId = serviceId; this.advertisedHost = advertisedHost; this.advertisementChannel = advertisementChannel; } @Override public void start() { - advertisement = advertisementChannel.advertise(advertisedHost); + try { + advertisementChannel.open(); + advertisement = advertisementChannel.advertise(serviceId, advertisedHost); + } catch (Exception failure) { + log.atSevere().withCause(failure) + .log("failed to advertise service"); + } } @Override @@ -42,10 +57,12 @@ public void stop() { currentAdvertisement.remove(); advertisement = null; } + advertisementChannel.close(); } @Override public String toString() { - return "AdvertisementHook(host=%s)".formatted(advertisedHost); + return "AdvertisementHook(serviceId=%s, host=%s, channel=%s)" + .formatted(serviceId, advertisedHost, advertisementChannel); } } \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/runtime/discovery/ConsulServiceAdvertisementChannel.java b/runtime/src/main/java/jsheets/runtime/discovery/ConsulServiceAdvertisementChannel.java deleted file mode 100644 index ec257e4..0000000 --- a/runtime/src/main/java/jsheets/runtime/discovery/ConsulServiceAdvertisementChannel.java +++ /dev/null @@ -1,82 +0,0 @@ -package jsheets.runtime.discovery; - -import java.util.Objects; - -import com.google.common.flogger.FluentLogger; -import com.google.common.flogger.MetadataKey; -import com.google.common.net.HostAndPort; - -import com.ecwid.consul.v1.ConsulClient; -import com.ecwid.consul.v1.agent.model.NewService; - -/** - * Implements a Consul - * backend for Service Discovery. - *

- * Consul discovery uses additional gRpc health checks - * that can only be used if the {@code Health} Feature is activated. - */ -public final class ConsulServiceAdvertisementChannel implements ServiceAdvertisementChannel { - private static final FluentLogger log = FluentLogger.forEnclosingClass(); - - public static ConsulServiceAdvertisementChannel forServiceId( - String serviceId, - ConsulClient client - ) { - Objects.requireNonNull(serviceId, "serviceId"); - Objects.requireNonNull(client, "client"); - return new ConsulServiceAdvertisementChannel(serviceId, client); - } - - private final String serviceId; - private final ConsulClient client; - - private ConsulServiceAdvertisementChannel(String serviceId, ConsulClient client) { - this.client = client; - this.serviceId = serviceId; - } - - private static final MetadataKey idKey = - MetadataKey.single("serviceId", String.class); - - @Override - public ServiceAdvertisement advertise(HostAndPort address) { - var service = createService(address); - log.atInfo() - .with(idKey, serviceId) - .log("advertising service in service discovery"); - client.agentServiceRegister(service); - return this::remove; - } - - private void remove() { - client.agentServiceDeregister(serviceId); - log.atInfo() - .with(idKey, serviceId) - .log("removing service discovery advertisement"); - } - - private NewService createService(HostAndPort address) { - var service = new NewService(); - service.setId(serviceId); - service.setPort(address.getPort()); - service.setAddress(address.getHost()); - service.setCheck(createCheck(address)); - return service; - } - - private static final String updateInterval = "10s"; - - private NewService.Check createCheck(HostAndPort address) { - var check = new NewService.Check(); - check.setGrpc(address.toString()); - check.setInterval(updateInterval); - return check; - } - - @Override - public String toString() { - return "ConsulServiceAdvertisement(serviceId=%s,client=%s)" - .formatted(serviceId, client); - } -} \ No newline at end of file diff --git a/runtime/src/main/java/jsheets/runtime/discovery/ServiceAdvertisementChannel.java b/runtime/src/main/java/jsheets/runtime/discovery/ServiceAdvertisementChannel.java index bda28e9..847aa3e 100644 --- a/runtime/src/main/java/jsheets/runtime/discovery/ServiceAdvertisementChannel.java +++ b/runtime/src/main/java/jsheets/runtime/discovery/ServiceAdvertisementChannel.java @@ -3,5 +3,7 @@ import com.google.common.net.HostAndPort; public interface ServiceAdvertisementChannel { - ServiceAdvertisement advertise(HostAndPort address); + ServiceAdvertisement advertise(String serviceId, HostAndPort address); + void open(); + void close(); } diff --git a/runtime/src/main/java/jsheets/runtime/discovery/ZookeeperServiceAdvertisementChannel.java b/runtime/src/main/java/jsheets/runtime/discovery/ZookeeperServiceAdvertisementChannel.java new file mode 100644 index 0000000..41cd984 --- /dev/null +++ b/runtime/src/main/java/jsheets/runtime/discovery/ZookeeperServiceAdvertisementChannel.java @@ -0,0 +1,81 @@ +package jsheets.runtime.discovery; + +import com.google.common.flogger.FluentLogger; +import com.google.common.net.HostAndPort; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.x.discovery.ServiceDiscovery; +import org.apache.curator.x.discovery.ServiceDiscoveryBuilder; +import org.apache.curator.x.discovery.ServiceInstance; +import org.apache.curator.x.discovery.ServiceType; + +public final class ZookeeperServiceAdvertisementChannel + implements ServiceAdvertisementChannel { + + private static final FluentLogger log = FluentLogger.forEnclosingClass(); + + public static ServiceAdvertisementChannel create(CuratorFramework curator) { + var discovery = ServiceDiscoveryBuilder.builder(Void.class) + .basePath("jsheets/services") + .client(curator) + .build(); + return new ZookeeperServiceAdvertisementChannel(discovery); + } + + private final ServiceDiscovery discovery; + + private ZookeeperServiceAdvertisementChannel(ServiceDiscovery discovery) { + this.discovery = discovery; + } + + @Override + public void open() { + try { + discovery.start(); + } catch (Exception failure) { + throw new RuntimeException(); + } + } + + @Override + public void close() { + try { + discovery.close(); + } catch (Exception failure) { + log.atWarning().withCause(failure).log("failed to close discovery"); + } + } + + @Override + public ServiceAdvertisement advertise(String serviceId, HostAndPort address) { + var service = createServiceInstance(serviceId, address); + try { + discovery.registerService(service); + } catch (Exception failedRegistration) { + throw new RuntimeException(failedRegistration); + } + return () -> { + try { + discovery.unregisterService(service); + } catch (Exception failure) { + log.atWarning().withCause(failure).log("failed to unregister service"); + } + }; + } + + private ServiceInstance createServiceInstance( + String serviceId, + HostAndPort address + ) { + return new ServiceInstance<>( + /* name */ "runtime", + /* id */ serviceId, + /* address */ address.getHost(), + /* port */ address.getPort(), + /* ssl port */ address.getPort(), + /* payload */ null, + /* registrationTimeUTC */ System.currentTimeMillis(), + /* serviceType */ ServiceType.DYNAMIC, + /* uri */ null + ); + } +} diff --git a/runtime/src/main/java/jsheets/runtime/evaluation/EvaluationModule.java b/runtime/src/main/java/jsheets/runtime/evaluation/EvaluationModule.java index 1a13f80..0c4e594 100644 --- a/runtime/src/main/java/jsheets/runtime/evaluation/EvaluationModule.java +++ b/runtime/src/main/java/jsheets/runtime/evaluation/EvaluationModule.java @@ -4,7 +4,6 @@ import com.google.inject.Provides; import com.google.inject.Singleton; -import javax.inject.Named; import jsheets.evaluation.EvaluationEngine; import jsheets.evaluation.sandbox.access.AccessGraph; import jsheets.evaluation.sandbox.validation.ForbiddenMemberFilter; @@ -12,10 +11,6 @@ import jsheets.evaluation.shell.environment.ExecutionEnvironment; import jsheets.evaluation.shell.environment.SandboxedEnvironment; import jsheets.evaluation.shell.environment.StandardEnvironment; -import jsheets.evaluation.shell.environment.inprocess.EmbeddedEnvironment; -import jsheets.evaluation.shell.environment.inprocess.InProcessExecutionControl; -import jsheets.evaluation.shell.execution.ExecutionMethod; -import jsheets.evaluation.shell.execution.ExhaustiveExecution; import jsheets.config.Config; import jsheets.evaluation.shell.execution.SystemBasedExecutionMethodFactory; diff --git a/runtime/src/main/resources/runtime/evaluation/sandbox/accessGraph.txt b/runtime/src/main/resources/runtime/evaluation/sandbox/accessGraph.txt index aaea75b..6afde38 100644 --- a/runtime/src/main/resources/runtime/evaluation/sandbox/accessGraph.txt +++ b/runtime/src/main/resources/runtime/evaluation/sandbox/accessGraph.txt @@ -1,2 +1,13 @@ -java.lang.Object -java.lang.Integer \ No newline at end of file +java.lang +!java.lang.Thread +!java.lang.reflect +!java.lang.invoke +java.lang.System +!java.lang.System#exit +!java.util.concurrent +java.text +java.time +java.math +java.io.Scanner +java.io.PrintStream#print +java.io.PrintStream#println diff --git a/server/build.gradle b/server/build.gradle index f22bafb..afdb6b6 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -26,6 +26,7 @@ ext { dependencies { implementation project(':protocol') implementation project(':evaluation') + implementation "org.apache.curator:curator-x-discovery:$curatorVersion" implementation "io.micrometer:micrometer-core:$micrometerVersion" implementation "org.mongodb:mongodb-driver-sync:$mongoDbDriverVersion" implementation "com.ecwid.consul:consul-api:$consuleClientVersion" @@ -39,6 +40,8 @@ dependencies { implementation "com.google.inject:guice:$guiceVersion" implementation "io.grpc:grpc-netty:$grpcVersion" implementation "io.grpc:grpc-stub:$grpcVersion" + implementation "io.grpc:grpc-core:$grpcVersion" + implementation "io.grpc:grpc-protobuf:$grpcVersion" testImplementation "org.junit.jupiter:junit-jupiter-api:$junitPlatformVersion" testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:$junitPlatformVersion" } diff --git a/server/src/main/java/jsheets/server/App.java b/server/src/main/java/jsheets/server/App.java index 21f05c7..1c23538 100644 --- a/server/src/main/java/jsheets/server/App.java +++ b/server/src/main/java/jsheets/server/App.java @@ -14,7 +14,10 @@ public static void main(String[] arguments) { } private static Injector configureInjector() { - return Guice.createInjector(ServerModule.create()); + return Guice.createInjector( + ConfigModule.create(), + ServerModule.create() + ); } private static void configureLogging() { diff --git a/server/src/main/java/jsheets/server/ConfigModule.java b/server/src/main/java/jsheets/server/ConfigModule.java new file mode 100644 index 0000000..cc50ec7 --- /dev/null +++ b/server/src/main/java/jsheets/server/ConfigModule.java @@ -0,0 +1,21 @@ +package jsheets.server; + +import com.google.inject.AbstractModule; +import com.google.inject.Provides; +import jsheets.config.Config; +import jsheets.config.EnvironmentConfig; + +public final class ConfigModule extends AbstractModule { + public static ConfigModule create() { + return new ConfigModule(); + } + + private ConfigModule() {} + + private static final String environmentPrefix = "JSHEETS"; + + @Provides + Config config() { + return EnvironmentConfig.prefixed(environmentPrefix).load(); + } +} diff --git a/server/src/main/java/jsheets/server/Server.java b/server/src/main/java/jsheets/server/Server.java index 0370162..0c94a59 100644 --- a/server/src/main/java/jsheets/server/Server.java +++ b/server/src/main/java/jsheets/server/Server.java @@ -19,17 +19,17 @@ public final class Server { private static final FluentLogger log = FluentLogger.forEnclosingClass(); - public record Config(int port) { } + public record Options(int port) { } private final Injector injector; - private final Config config; + private final Options options; private final AtomicReference runningServer = new AtomicReference<>(null); @Inject - Server(Injector injector, Config config) { + Server(Injector injector, Options options) { this.injector = injector; - this.config = config; + this.options = options; } public void start() { @@ -38,7 +38,7 @@ public void start() { throw new IllegalStateException("already running"); } log.atInfo().log("starting..."); - server.start(config.port()); + server.start(options.port()); } public void stop() { diff --git a/server/src/main/java/jsheets/server/ServerModule.java b/server/src/main/java/jsheets/server/ServerModule.java index 48cbd1e..7e09a8a 100644 --- a/server/src/main/java/jsheets/server/ServerModule.java +++ b/server/src/main/java/jsheets/server/ServerModule.java @@ -28,8 +28,8 @@ protected void configure() { private static final int defaultPort = 8080; @Provides - Server.Config createConfig() { - return new Server.Config( + Server.Options createConfig() { + return new Server.Options( readIntFromEnvironment("JSHEETS_SERVER_PORT").orElse(defaultPort) ); } diff --git a/server/src/main/java/jsheets/server/evaluation/EvaluationConnection.java b/server/src/main/java/jsheets/server/evaluation/EvaluationConnection.java index 6c20899..e35f1b6 100644 --- a/server/src/main/java/jsheets/server/evaluation/EvaluationConnection.java +++ b/server/src/main/java/jsheets/server/evaluation/EvaluationConnection.java @@ -138,20 +138,38 @@ private void receiveStart(WsContext context, StartEvaluationRequest request) { establishConnection(context, request); } - private static final CloseStatus CANCELLED = + private static final CloseStatus cancelledStatus = new CloseStatus(HttpStatus.BAD_REQUEST_400, "cancelled"); + private static final CloseStatus runtimeUnavailableStatus = + new CloseStatus(HttpStatus.ENHANCE_YOUR_CALM_420, "runtime unavailable"); + private void establishConnection( WsContext context, StartEvaluationRequest request ) { var listener = new UpstreamListener(context); - var upstream = engine.start(request, listener); + Evaluation upstream; + try { + upstream = engine.start(request, listener); + } catch (Throwable failure) { + reportFailedStart(context, failure); + return; + } if (!completeConnecting(upstream)) { - ensureSessionIsClosed(context.session, CANCELLED); + ensureSessionIsClosed(context.session, cancelledStatus); } } + private void reportFailedStart(WsContext context, Throwable failure) { + log.atWarning() + .with(sessionIdMetadata, context.getSessionId()) + .withCause(failure) + .atMostEvery(5, TimeUnit.SECONDS) + .log("failed to start evaluation"); + ensureSessionIsClosed(context.session, cancelledStatus); + } + private boolean completeConnecting(Evaluation upstream) { if (!stage.compareAndSet(Stage.Connecting, Stage.Evaluating)) { log.atWarning().log("the evaluation was terminated in the connecting stage"); @@ -209,6 +227,8 @@ public String toString() { private void ensureSessionIsClosed(Session session, CloseStatus status) { // close does not throw an exception if the session is already closed + stage.set(Stage.Terminated); + closeHook.run(); session.close(status); } diff --git a/server/src/main/java/jsheets/server/evaluation/EvaluationModule.java b/server/src/main/java/jsheets/server/evaluation/EvaluationModule.java index 9a601b0..9547997 100644 --- a/server/src/main/java/jsheets/server/evaluation/EvaluationModule.java +++ b/server/src/main/java/jsheets/server/evaluation/EvaluationModule.java @@ -1,15 +1,27 @@ package jsheets.server.evaluation; -import java.util.concurrent.Executors; - +import com.google.common.flogger.FluentLogger; import com.google.inject.AbstractModule; import com.google.inject.Provides; import com.google.inject.Singleton; +import jsheets.config.Config; import jsheets.evaluation.EvaluationEngine; import jsheets.evaluation.shell.ShellEvaluationEngine; +import jsheets.server.evaluation.client.PooledEvaluationEngine; +import jsheets.server.evaluation.client.ZookeeperEngineDiscovery; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.ExponentialBackoffRetry; +import org.apache.curator.x.discovery.ServiceDiscoveryBuilder; +import org.apache.curator.x.discovery.ServiceProvider; + +import java.util.Optional; +import java.util.concurrent.Executors; public final class EvaluationModule extends AbstractModule { + private static final FluentLogger log = FluentLogger.forEnclosingClass(); + public static EvaluationModule create() { return new EvaluationModule(); } @@ -18,9 +30,68 @@ private EvaluationModule() {} @Provides @Singleton - EvaluationEngine evaluationEngine() { + EvaluationEngine evaluationEngine( + Optional curatorBinding + ) { + return curatorBinding.map(this::createRemoteEvaluationEngine) + .orElseGet(this::createEmbeddedEvaluationEngine); + } + + private EvaluationEngine createRemoteEvaluationEngine(CuratorFramework client) { + var provider = createServiceProvider(client); + try { + provider.start(); + } catch (Exception failure) { + log.atSevere().withCause(failure).log("failed to start service discovery"); + } + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + try { + provider.close(); + } catch (Exception failure) { + log.atWarning().withCause(failure).log("failed to close service discovery"); + } + })); + return PooledEvaluationEngine.of(ZookeeperEngineDiscovery.create(provider)); + } + + private EvaluationEngine createEmbeddedEvaluationEngine() { return ShellEvaluationEngine.newBuilder() .useWorkerPool(Executors.newCachedThreadPool()) .create(); } + + private ServiceProvider createServiceProvider(CuratorFramework curator) { + return ServiceDiscoveryBuilder.builder(Void.class) + .client(curator) + .basePath("/jsheets/services") + .build() + .serviceProviderBuilder() + .serviceName("runtime") + .build(); + } + + private static final Config.Key connectionStringKey + = Config.Key.ofString("zookeeper.connectionString"); + + private static final Config.Key connectionBackoffKey + = Config.Key.ofInt("zookeeper.connectBackoff"); + + private static final int defaultBackoff = 1000; + private static final int retryLimit = 3; + + @Provides + @Singleton + Optional curatorFramework(Config config) { + return connectionStringKey.in(config).orNone() + .map(connectionString -> { + int backoff = connectionBackoffKey.in(config).or(defaultBackoff); + var client = CuratorFrameworkFactory.newClient( + connectionString, + new ExponentialBackoffRetry(backoff, retryLimit) + ); + client.start(); + Runtime.getRuntime().addShutdownHook(new Thread(client::close)); + return client; + }); + } } \ No newline at end of file diff --git a/server/src/main/java/jsheets/server/evaluation/client/ConsulServiceDiscovery.java b/server/src/main/java/jsheets/server/evaluation/client/ConsulServiceDiscovery.java deleted file mode 100644 index 0a8fc7d..0000000 --- a/server/src/main/java/jsheets/server/evaluation/client/ConsulServiceDiscovery.java +++ /dev/null @@ -1,19 +0,0 @@ -package jsheets.server.evaluation.client; - -import java.util.Optional; - -import com.ecwid.consul.v1.ConsulClient; -import jsheets.evaluation.EvaluationEngine; - -public final class ConsulServiceDiscovery implements EnginePool { - private final ConsulClient client; - - private ConsulServiceDiscovery(ConsulClient client) { - this.client = client; - } - - @Override - public Optional select() { - return Optional.empty(); - } -} \ No newline at end of file diff --git a/server/src/main/java/jsheets/server/evaluation/client/PooledEvaluationEngine.java b/server/src/main/java/jsheets/server/evaluation/client/PooledEvaluationEngine.java new file mode 100644 index 0000000..6f49d3b --- /dev/null +++ b/server/src/main/java/jsheets/server/evaluation/client/PooledEvaluationEngine.java @@ -0,0 +1,34 @@ +package jsheets.server.evaluation.client; + +import jsheets.StartEvaluationRequest; +import jsheets.evaluation.Evaluation; +import jsheets.evaluation.EvaluationEngine; + +import javax.inject.Inject; +import java.util.Objects; + +public final class PooledEvaluationEngine implements EvaluationEngine { + public static PooledEvaluationEngine of(EnginePool pool) { + Objects.requireNonNull(pool, "pool"); + return new PooledEvaluationEngine(pool); + } + + private final EnginePool pool; + + @Inject + PooledEvaluationEngine(EnginePool pool) { + this.pool = pool; + } + + @Override + public Evaluation start(StartEvaluationRequest request, Evaluation.Listener listener) { + return pool.select() + .map(engine -> engine.start(request, listener)) + .orElseGet(() -> reportNoEngineFound(listener)); + } + + private Evaluation reportNoEngineFound(Evaluation.Listener listener) { + listener.close(); + return () -> {}; + } +} diff --git a/server/src/main/java/jsheets/server/evaluation/client/SnippetRuntimeEngine.java b/server/src/main/java/jsheets/server/evaluation/client/SnippetRuntimeEngine.java index edcc68f..97d6d9b 100644 --- a/server/src/main/java/jsheets/server/evaluation/client/SnippetRuntimeEngine.java +++ b/server/src/main/java/jsheets/server/evaluation/client/SnippetRuntimeEngine.java @@ -21,7 +21,7 @@ * Client side {@link EvaluationEngine} that connects to a * {@code SnippetRuntime} to evaluate snippets. */ -final class SnippetRuntimeEngine implements EvaluationEngine { +public final class SnippetRuntimeEngine implements EvaluationEngine { private static final FluentLogger log = FluentLogger.forEnclosingClass(); public static SnippetRuntimeEngine forChannel(Channel channel) { diff --git a/server/src/main/java/jsheets/server/evaluation/client/ZookeeperEngineDiscovery.java b/server/src/main/java/jsheets/server/evaluation/client/ZookeeperEngineDiscovery.java new file mode 100644 index 0000000..9c82833 --- /dev/null +++ b/server/src/main/java/jsheets/server/evaluation/client/ZookeeperEngineDiscovery.java @@ -0,0 +1,43 @@ +package jsheets.server.evaluation.client; + +import com.google.common.flogger.FluentLogger; +import io.grpc.ManagedChannelBuilder; +import jsheets.evaluation.EvaluationEngine; +import org.apache.curator.x.discovery.ServiceProvider; + +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +// TODO: Pool Connections +public final class ZookeeperEngineDiscovery implements EnginePool { + private static final FluentLogger log = FluentLogger.forEnclosingClass(); + + public static ZookeeperEngineDiscovery create(ServiceProvider services) { + Objects.requireNonNull(services, "services"); + return new ZookeeperEngineDiscovery(services); + } + + private final ServiceProvider services; + + private ZookeeperEngineDiscovery(ServiceProvider services) { + this.services = services; + } + + @Override + public Optional select() { + try { + var service = services.getInstance(); + var channel = ManagedChannelBuilder.forAddress(service.getAddress(), service.getPort()) + .usePlaintext() + .build(); + return Optional.of(SnippetRuntimeEngine.forChannel(channel)); + } catch (Exception failure) { + log.atWarning() + .withCause(failure) + .atMostEvery(5, TimeUnit.SECONDS) + .log("failed to find runtime instance"); + return Optional.empty(); + } + } +} diff --git a/website/src/client/index.ts b/website/src/client/index.ts index 2d630dc..239be66 100644 --- a/website/src/client/index.ts +++ b/website/src/client/index.ts @@ -57,6 +57,7 @@ export default class Client { client.onerror = error => { console.log({error}) + listener.onEnd() } client.onopen = () => { From 93737f40088c74ff089fc364f775e031fd867b33 Mon Sep 17 00:00:00 2001 From: Merlin Osayimwen Date: Sat, 16 Oct 2021 18:44:13 +0200 Subject: [PATCH 13/14] Create Forked Execution Environment --- .../shell/environment/ClassFileStore.java | 8 + .../environment/ClassFileStoreLoader.java} | 70 ++----- .../fork/ForkedExecutionControl.java | 129 +++++++++++++ .../fork/ForkedExecutionEnvironment.java | 40 ++++ .../fork/ForkingExecutionControlProvider.java | 171 ++++++++++++++++++ .../environment/fork/RemoteInterrupt.java | 94 ++++++++++ .../sandbox/SandboxClassFileCheck.java | 38 ++++ .../{ => sandbox}/SandboxedEnvironment.java | 16 +- .../sandbox/SandboxExecutionTest.java | 2 +- .../java/jsheets/runtime/ConfigModule.java | 4 +- ...ource.java => EvaluationConfigSource.java} | 55 ++++-- .../runtime/evaluation/EvaluationModule.java | 25 ++- .../evaluation/fork/virtualMachineOptions.txt | 3 + 13 files changed, 576 insertions(+), 79 deletions(-) create mode 100644 evaluation/src/main/java/jsheets/evaluation/shell/environment/ClassFileStore.java rename evaluation/src/main/java/jsheets/evaluation/{sandbox/SandboxLoader.java => shell/environment/ClassFileStoreLoader.java} (79%) create mode 100644 evaluation/src/main/java/jsheets/evaluation/shell/environment/fork/ForkedExecutionControl.java create mode 100644 evaluation/src/main/java/jsheets/evaluation/shell/environment/fork/ForkedExecutionEnvironment.java create mode 100644 evaluation/src/main/java/jsheets/evaluation/shell/environment/fork/ForkingExecutionControlProvider.java create mode 100644 evaluation/src/main/java/jsheets/evaluation/shell/environment/fork/RemoteInterrupt.java create mode 100644 evaluation/src/main/java/jsheets/evaluation/shell/environment/sandbox/SandboxClassFileCheck.java rename evaluation/src/main/java/jsheets/evaluation/shell/environment/{ => sandbox}/SandboxedEnvironment.java (63%) rename runtime/src/main/java/jsheets/runtime/evaluation/{SandboxConfigSource.java => EvaluationConfigSource.java} (50%) create mode 100644 runtime/src/main/resources/runtime/evaluation/fork/virtualMachineOptions.txt diff --git a/evaluation/src/main/java/jsheets/evaluation/shell/environment/ClassFileStore.java b/evaluation/src/main/java/jsheets/evaluation/shell/environment/ClassFileStore.java new file mode 100644 index 0000000..b87c5f9 --- /dev/null +++ b/evaluation/src/main/java/jsheets/evaluation/shell/environment/ClassFileStore.java @@ -0,0 +1,8 @@ +package jsheets.evaluation.shell.environment; + +import jdk.jshell.spi.ExecutionControl; + +public interface ClassFileStore { + void redefine(ExecutionControl.ClassBytecodes[] bytecodes); + void load(ExecutionControl.ClassBytecodes[] bytecodes); +} diff --git a/evaluation/src/main/java/jsheets/evaluation/sandbox/SandboxLoader.java b/evaluation/src/main/java/jsheets/evaluation/shell/environment/ClassFileStoreLoader.java similarity index 79% rename from evaluation/src/main/java/jsheets/evaluation/sandbox/SandboxLoader.java rename to evaluation/src/main/java/jsheets/evaluation/shell/environment/ClassFileStoreLoader.java index 9ceb117..6fb9211 100644 --- a/evaluation/src/main/java/jsheets/evaluation/sandbox/SandboxLoader.java +++ b/evaluation/src/main/java/jsheets/evaluation/shell/environment/ClassFileStoreLoader.java @@ -1,4 +1,4 @@ -package jsheets.evaluation.sandbox; +package jsheets.evaluation.shell.environment; import java.io.ByteArrayInputStream; import java.io.File; @@ -15,7 +15,6 @@ import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.Enumeration; @@ -27,30 +26,26 @@ import jdk.jshell.execution.LoaderDelegate; import jdk.jshell.spi.ExecutionControl; -import jsheets.evaluation.sandbox.validation.Analysis; -import jsheets.evaluation.sandbox.validation.Rule; -public final class SandboxLoader implements LoaderDelegate { +public final class ClassFileStoreLoader implements LoaderDelegate { // This class is based on Java's loader for DirectExecution. - - public static SandboxLoader create(Collection rules) { - Objects.requireNonNull(rules, "rules"); - return new SandboxLoader(rules); + public static ClassFileStoreLoader of(ClassFileStore store) { + Objects.requireNonNull(store, "store"); + return new ClassFileStoreLoader(store); } - private final SandboxLoader.RemoteClassLoader loader; private final Map> types = new HashMap<>(); - private final Collection rules; + private final ClassFileStore store; + private final RemoteClassLoader remote = new RemoteClassLoader(); - private SandboxLoader(Collection rules) { - this.loader = new RemoteClassLoader(); - this.rules = rules; + private ClassFileStoreLoader(ClassFileStore store) { + this.store = store; } public Runnable install() { var thread = Thread.currentThread(); var previousLoader = thread.getContextClassLoader(); - thread.setContextClassLoader(loader); + thread.setContextClassLoader(remote); return () -> thread.setContextClassLoader(previousLoader); } @@ -58,39 +53,11 @@ public Runnable install() { public void load(ExecutionControl.ClassBytecodes[] binaries) throws ExecutionControl.ClassInstallException { - loadBinaries(binaries); - preload(binaries); - } - - private void loadBinaries(ExecutionControl.ClassBytecodes[] binaries) - throws ExecutionControl.ClassInstallException - { - var analysis = Analysis.create(); - var check = SandboxBytecodeCheck.withRules(rules); - loadBinariesWithAnalysis(analysis, check, binaries); - checkAnalysisReport(analysis); - } - - private void checkAnalysisReport(Analysis analysis) { - analysis.reportViolations(); - } - - private void loadBinariesWithAnalysis( - Analysis analysis, - SandboxBytecodeCheck check, - ExecutionControl.ClassBytecodes[] binaries - ) throws ExecutionControl.ClassInstallException{ - try { - for (var binary : binaries) { - check.run(analysis, binary.bytecodes()); - loader.declare(binary.name(), binary.bytecodes()); - } - } catch (Throwable failure) { - throw new ExecutionControl.ClassInstallException( - "declare: " + failure.getMessage(), - new boolean[0] - ); + store.load(binaries); + for (var binary : binaries) { + remote.declare(binary.name(), binary.bytecodes()); } + preload(binaries); } private void preload(ExecutionControl.ClassBytecodes[] binaries) @@ -100,7 +67,7 @@ private void preload(ExecutionControl.ClassBytecodes[] binaries) try { for ( int index = 0; index < binaries.length; ++index ) { var code = binaries[index]; - var type = loader.loadClass(code.name()); + var type = remote.loadClass(code.name()); types.put(code.name(), type); loaded[index] = true; preload(type); @@ -118,8 +85,9 @@ private void preload(Class type) { @Override public void classesRedefined(ExecutionControl.ClassBytecodes[] binaries) { + store.redefine(binaries); for (var binary : binaries) { - loader.declare(binary.name(), binary.bytecodes()); + remote.declare(binary.name(), binary.bytecodes()); } } @@ -127,7 +95,7 @@ public void classesRedefined(ExecutionControl.ClassBytecodes[] binaries) { public void addToClasspath(String classPath) throws ExecutionControl.InternalException { try { for (var path : classPath.split(File.pathSeparator)) { - loader.addURL(new File(path).toURI().toURL()); + remote.addURL(new File(path).toURI().toURL()); } } catch (Exception failure) { throw new ExecutionControl.InternalException(failure.toString()); @@ -192,7 +160,7 @@ private URL lookupResource(String name) { return new URL( /* context */ null, new URI("jshell", null, "/" + name, null).toString(), - new ResourceUrlStreamHandler(name) + new RemoteClassLoader.ResourceUrlStreamHandler(name) ); } catch (MalformedURLException | URISyntaxException failure) { throw new InternalError(failure); diff --git a/evaluation/src/main/java/jsheets/evaluation/shell/environment/fork/ForkedExecutionControl.java b/evaluation/src/main/java/jsheets/evaluation/shell/environment/fork/ForkedExecutionControl.java new file mode 100644 index 0000000..aab6520 --- /dev/null +++ b/evaluation/src/main/java/jsheets/evaluation/shell/environment/fork/ForkedExecutionControl.java @@ -0,0 +1,129 @@ +package jsheets.evaluation.shell.environment.fork; + +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import com.google.common.flogger.FluentLogger; + +import com.sun.jdi.VMDisconnectedException; +import com.sun.jdi.VirtualMachine; +import jdk.jshell.execution.JdiExecutionControl; +import jsheets.evaluation.shell.environment.ClassFileStore; + +/* + * This class is based on the JdiDefaultExecutionControl and is highly + * coupled to the remote agent implementation. + */ +public final class ForkedExecutionControl extends JdiExecutionControl { + private static final FluentLogger log = FluentLogger.forEnclosingClass(); + + private VirtualMachine machine; + private Process process; + + private final Lock stopLock = new ReentrantLock(); + private boolean userCodeRunning = false; + + private final String remoteAgentClass; + private final ClassFileStore classFileStore; + + ForkedExecutionControl( + ObjectOutput output, + ObjectInput input, + VirtualMachine machine, + Process process, + String remoteAgentClass, + ClassFileStore classFileStore + ) { + super(output, input); + this.machine = machine; + this.process = process; + this.remoteAgentClass = remoteAgentClass; + this.classFileStore = classFileStore; + } + + @Override + protected synchronized VirtualMachine vm() throws EngineTerminationException { + if (machine == null) { + throw new EngineTerminationException("virtual machine is closed"); + } + return machine; + } + + @Override + public void load(ClassBytecodes[] bytecodes) + throws ClassInstallException, NotImplementedException, EngineTerminationException + { + classFileStore.load(bytecodes); + super.load(bytecodes); + } + + @Override + public void redefine(ClassBytecodes[] bytecodes) + throws ClassInstallException, EngineTerminationException + { + classFileStore.redefine(bytecodes); + super.redefine(bytecodes); + } + + @Override + public String invoke(String classname, String methodName) + throws RunException, EngineTerminationException, InternalException + { + String result; + updateUserCodeRunning(true); + try { + result = super.invoke(classname, methodName); + } finally { + updateUserCodeRunning(false); + } + return result; + } + + private void updateUserCodeRunning(boolean target) { + stopLock.lock(); + try { + userCodeRunning = target; + } finally { + stopLock.unlock(); + } + } + + @Override + public void stop() throws EngineTerminationException, InternalException { + stopLock.lock(); + try { + if (userCodeRunning) { + new RemoteInterrupt(vm(), remoteAgentClass).runInSuspendedMode(); + } + } finally { + stopLock.unlock(); + } + } + + + @Override + public void close() { + super.close(); + disposeMachine(); + } + + synchronized void disposeMachine() { + try { + if (machine != null) { + machine.dispose(); + machine = null; + } + } catch (VMDisconnectedException alreadyClosed) { + log.atFiner().withCause(alreadyClosed).log("remote is already closed"); + } catch (Throwable failure) { + log.atFine().withCause(failure).log("failed to dispose remote"); + } finally { + if (process != null) { + process.destroy(); + process = null; + } + } + } +} \ No newline at end of file diff --git a/evaluation/src/main/java/jsheets/evaluation/shell/environment/fork/ForkedExecutionEnvironment.java b/evaluation/src/main/java/jsheets/evaluation/shell/environment/fork/ForkedExecutionEnvironment.java new file mode 100644 index 0000000..4511e5a --- /dev/null +++ b/evaluation/src/main/java/jsheets/evaluation/shell/environment/fork/ForkedExecutionEnvironment.java @@ -0,0 +1,40 @@ +package jsheets.evaluation.shell.environment.fork; + +import java.util.Collection; +import java.util.Objects; + +import jdk.jshell.spi.ExecutionControlProvider; +import jsheets.evaluation.shell.environment.ClassFileStore; +import jsheets.evaluation.shell.environment.ExecutionEnvironment; + +public final class ForkedExecutionEnvironment implements ExecutionEnvironment { + public static ForkedExecutionEnvironment create( + ClassFileStore store, + Collection virtualMachineOptions + ) { + Objects.requireNonNull(store, "store"); + Objects.requireNonNull(virtualMachineOptions, "virtualMachineOptions"); + return new ForkedExecutionEnvironment(store, virtualMachineOptions); + } + + private final ClassFileStore store; + private final Collection virtualMachineOptions; + + private ForkedExecutionEnvironment( + ClassFileStore store, + Collection virtualMachineOptions + ) { + this.store = store; + this.virtualMachineOptions = virtualMachineOptions; + } + + @Override + public ExecutionControlProvider control(String name) { + return ForkingExecutionControlProvider.create(virtualMachineOptions, store); + } + + @Override + public Installation install() { + return () -> {}; + } +} \ No newline at end of file diff --git a/evaluation/src/main/java/jsheets/evaluation/shell/environment/fork/ForkingExecutionControlProvider.java b/evaluation/src/main/java/jsheets/evaluation/shell/environment/fork/ForkingExecutionControlProvider.java new file mode 100644 index 0000000..74ca3cb --- /dev/null +++ b/evaluation/src/main/java/jsheets/evaluation/shell/environment/fork/ForkingExecutionControlProvider.java @@ -0,0 +1,171 @@ +package jsheets.evaluation.shell.environment.fork; + +import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.time.Duration; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.BiFunction; + +import com.sun.jdi.VirtualMachine; +import jdk.jshell.execution.JdiInitiator; +import jdk.jshell.execution.RemoteExecutionControl; +import jdk.jshell.execution.Util; +import jdk.jshell.spi.ExecutionControl; +import jdk.jshell.spi.ExecutionControlProvider; +import jdk.jshell.spi.ExecutionEnv; +import jsheets.evaluation.shell.environment.ClassFileStore; + +import static jdk.jshell.execution.Util.remoteInputOutput; + +public final class ForkingExecutionControlProvider + implements ExecutionControlProvider { + + private static final Duration defaultTimeout = Duration.ofMillis(3000); + + public static ForkingExecutionControlProvider create( + Collection rawVirtualMachineOptions, + ClassFileStore classFileStore + ) { + Objects.requireNonNull(classFileStore, "classFileStore"); + Objects.requireNonNull(rawVirtualMachineOptions, "rawVirtualMachineOptions"); + var host = InetAddress.getLoopbackAddress().getHostName(); + return new ForkingExecutionControlProvider( + host, + defaultTimeout, + List.copyOf(rawVirtualMachineOptions), + classFileStore + ); + } + + private final String host; + private final Duration timeout; + private final List rawVirtualMachineOptions; + private final ClassFileStore classFileStore; + + private ForkingExecutionControlProvider( + String host, + Duration timeout, + List rawVirtualMachineOptions, + ClassFileStore classFileStore + ) { + this.host = host; + this.timeout = timeout; + this.rawVirtualMachineOptions = rawVirtualMachineOptions; + this.classFileStore = classFileStore; + } + + @Override + public String name() { + return "forking-execution-control-provider"; + } + + @Override + public Map defaultParameters() { + return Map.of(); + } + + @Override + public ExecutionControl generate( + ExecutionEnv environment, + Map specialParameters + ) throws IOException { + return create(environment); + } + + record Box(VirtualMachine machine, Process process) {} + + private static final String remoteAgentClassName = + RemoteExecutionControl.class.getName(); + + private Box initiate(int port) { + var initiator = new JdiInitiator( + /* port */ port, + /* options */ rawVirtualMachineOptions, + /* remoteAgentClassName */ remoteAgentClassName, + /* controlledLaunch */ true, + /* host */ host, + /* timeout */ (int) timeout.toMillis(), + /* connectorOptions*/ Map.of() + ); + return new Box( + initiator.vm(), + initiator.process() + ); + } + + private static final int backlog = 1; + + ExecutionControl create(ExecutionEnv environment) throws IOException { + var address = InetAddress.getLoopbackAddress(); + try (var listener = new ServerSocket(0, backlog, address)) { + listener.setSoTimeout((int) timeout.toMillis()); + var box = initiate(listener.getLocalPort()); + return accept(listener, environment, box); + } + } + + private ExecutionControl accept( + ServerSocket listener, + ExecutionEnv environment, + Box box + ) throws IOException { + var socket = listener.accept(); + var output = socket.getOutputStream(); + return remoteInputOutput( + socket.getInputStream(), + output, + createOutputs(environment), + createInputs(environment), + createControl(box, environment) + ); + } + + private BiFunction createControl( + Box box, + ExecutionEnv environment + ) { + return (input, output) -> { + var control = new ForkedExecutionControl( + output, + input, + box.machine(), + box.process(), + remoteAgentClassName, + classFileStore + ); + registerCloseHooks(box.machine(), environment, control); + return control; + }; + } + + private Map createOutputs(ExecutionEnv environment) { + return Map.of( + "out", environment.userOut(), + "err", environment.userErr() + ); + } + + private Map createInputs(ExecutionEnv environment) { + return Map.of("in", environment.userIn()); + } + + private void registerCloseHooks( + VirtualMachine machine, + ExecutionEnv environment, + ForkedExecutionControl control + ) { + Util.detectJdiExitEvent(machine, event -> { + environment.closeDown(); + control.disposeMachine(); + }); + } +} diff --git a/evaluation/src/main/java/jsheets/evaluation/shell/environment/fork/RemoteInterrupt.java b/evaluation/src/main/java/jsheets/evaluation/shell/environment/fork/RemoteInterrupt.java new file mode 100644 index 0000000..4d05c7d --- /dev/null +++ b/evaluation/src/main/java/jsheets/evaluation/shell/environment/fork/RemoteInterrupt.java @@ -0,0 +1,94 @@ +package jsheets.evaluation.shell.environment.fork; + +import java.util.Objects; +import java.util.Set; + +import com.google.common.flogger.FluentLogger; + +import com.sun.jdi.BooleanValue; +import com.sun.jdi.ObjectReference; +import com.sun.jdi.StackFrame; +import com.sun.jdi.ThreadReference; +import com.sun.jdi.VirtualMachine; +import jdk.jshell.spi.ExecutionControl; + +final class RemoteInterrupt { + private static final FluentLogger log = FluentLogger.forEnclosingClass(); + + private final VirtualMachine virtualMachine; + private final String remoteAgentClass; + + RemoteInterrupt(VirtualMachine virtualMachine, String remoteAgentClass) { + this.virtualMachine = virtualMachine; + this.remoteAgentClass = remoteAgentClass; + } + + public void runInSuspendedMode() throws ExecutionControl.InternalException { + virtualMachine.suspend(); + try { + run(); + } catch (Exception failure) { + throw new ExecutionControl.InternalException( + "failed to stop remote execution: %s".formatted(failure) + ); + } finally { + virtualMachine.resume(); + } + } + + public void run() throws Exception { + for (var thread : virtualMachine.allThreads()) { + if (shouldIgnoreThread(thread)) { + continue; + } + for (var frame : thread.frames()) { + if (isRemoteAgentFrame(frame)) { + var instance = frame.thisObject(); + Objects.requireNonNull(instance, "frame instance is null"); + closeRemoteAgentThread(thread, instance); + } + } + } + } + + private static final Set agentMethods = Set.of("invoke", "varValue"); + + private boolean isRemoteAgentFrame(StackFrame frame) { + var location = frame.location(); + if (!remoteAgentClass.equals(location.declaringType().name())) { + return false; + } + var methodName = location.method().name(); + return agentMethods.contains(methodName); + } + + /* Threads created by user code can and should be ignored */ + private boolean shouldIgnoreThread(ThreadReference thread) { + log.atInfo().log("not ignoring thread " + thread); + return false; + } + + /* + * Closes the remote agent thread by manipulating the fields that are + * controlling its execution state and throwing an asynchronous exception. + * This code is highly coupled with the agent implementation running on the + * remote end. + */ + private void closeRemoteAgentThread( + ThreadReference thread, + ObjectReference frame + ) throws Exception { + var inClientCodeField = frame.referenceType().fieldByName("inClientCode"); + var expectingStopField = frame.referenceType().fieldByName("expectingStop"); + var stopExceptionField = frame.referenceType().fieldByName("stopException"); + var inClientCode = (BooleanValue) frame.getValue(inClientCodeField); + if (inClientCode.value()) { + frame.setValue(expectingStopField, virtualMachine.mirrorOf(true)); + var stopInstance = (ObjectReference) frame.getValue(stopExceptionField); + virtualMachine.resume(); + log.atFine().log("attempting to stop the client code execution"); + thread.stop(stopInstance); + frame.setValue(expectingStopField, virtualMachine.mirrorOf(false)); + } + } +} \ No newline at end of file diff --git a/evaluation/src/main/java/jsheets/evaluation/shell/environment/sandbox/SandboxClassFileCheck.java b/evaluation/src/main/java/jsheets/evaluation/shell/environment/sandbox/SandboxClassFileCheck.java new file mode 100644 index 0000000..b2c0785 --- /dev/null +++ b/evaluation/src/main/java/jsheets/evaluation/shell/environment/sandbox/SandboxClassFileCheck.java @@ -0,0 +1,38 @@ +package jsheets.evaluation.shell.environment.sandbox; + +import java.util.Collection; +import java.util.Objects; + +import jdk.jshell.spi.ExecutionControl; +import jsheets.evaluation.sandbox.SandboxBytecodeCheck; +import jsheets.evaluation.sandbox.validation.Analysis; +import jsheets.evaluation.sandbox.validation.Rule; +import jsheets.evaluation.shell.environment.ClassFileStore; + +public final class SandboxClassFileCheck implements ClassFileStore { + public static SandboxClassFileCheck of(Collection rules) { + Objects.requireNonNull(rules, "rules"); + return new SandboxClassFileCheck(rules); + } + + private final Collection rules; + + private SandboxClassFileCheck(Collection rules) { + this.rules = rules; + } + + @Override + public void redefine(ExecutionControl.ClassBytecodes[] bytecodes) { + var analysis = Analysis.create(); + var check = SandboxBytecodeCheck.withRules(rules); + for (var binary : bytecodes) { + check.run(analysis, binary.bytecodes()); + } + analysis.reportViolations(); + } + + @Override + public void load(ExecutionControl.ClassBytecodes[] bytecodes) { + + } +} \ No newline at end of file diff --git a/evaluation/src/main/java/jsheets/evaluation/shell/environment/SandboxedEnvironment.java b/evaluation/src/main/java/jsheets/evaluation/shell/environment/sandbox/SandboxedEnvironment.java similarity index 63% rename from evaluation/src/main/java/jsheets/evaluation/shell/environment/SandboxedEnvironment.java rename to evaluation/src/main/java/jsheets/evaluation/shell/environment/sandbox/SandboxedEnvironment.java index b308b48..82f6408 100644 --- a/evaluation/src/main/java/jsheets/evaluation/shell/environment/SandboxedEnvironment.java +++ b/evaluation/src/main/java/jsheets/evaluation/shell/environment/sandbox/SandboxedEnvironment.java @@ -1,10 +1,12 @@ -package jsheets.evaluation.shell.environment; +package jsheets.evaluation.shell.environment.sandbox; import jdk.jshell.execution.DirectExecutionControl; import jdk.jshell.spi.ExecutionControl; import jdk.jshell.spi.ExecutionControlProvider; import jdk.jshell.spi.ExecutionEnv; -import jsheets.evaluation.sandbox.SandboxLoader; +import jsheets.evaluation.shell.environment.ClassFileStore; +import jsheets.evaluation.shell.environment.ClassFileStoreLoader; +import jsheets.evaluation.shell.environment.ExecutionEnvironment; import jsheets.evaluation.sandbox.validation.Rule; import java.util.Collection; @@ -17,18 +19,18 @@ public final class SandboxedEnvironment public static SandboxedEnvironment create(Collection rules) { Objects.requireNonNull(rules); - return new SandboxedEnvironment(() -> SandboxLoader.create(rules)); + return new SandboxedEnvironment(() -> SandboxClassFileCheck.of(rules)); } - private final Supplier loader; + private final Supplier loader; - private SandboxedEnvironment(Supplier loader) { + private SandboxedEnvironment(Supplier loader) { this.loader = loader; } @Override public Installation install() { - return loader.get().install()::run; + return () -> {}; } @Override @@ -46,6 +48,6 @@ public ExecutionControl generate( ExecutionEnv environment, Map parameters ) { - return new DirectExecutionControl(loader.get()); + return new DirectExecutionControl(ClassFileStoreLoader.of(loader.get())); } } diff --git a/evaluation/src/test/java/jsheets/evaluation/sandbox/SandboxExecutionTest.java b/evaluation/src/test/java/jsheets/evaluation/sandbox/SandboxExecutionTest.java index 66564b3..7d648cb 100644 --- a/evaluation/src/test/java/jsheets/evaluation/sandbox/SandboxExecutionTest.java +++ b/evaluation/src/test/java/jsheets/evaluation/sandbox/SandboxExecutionTest.java @@ -4,7 +4,7 @@ import jsheets.evaluation.sandbox.access.AccessGraph; import jsheets.evaluation.sandbox.validation.Analysis; import jsheets.evaluation.sandbox.validation.ForbiddenMemberFilter; -import jsheets.evaluation.shell.environment.SandboxedEnvironment; +import jsheets.evaluation.shell.environment.sandbox.SandboxedEnvironment; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/runtime/src/main/java/jsheets/runtime/ConfigModule.java b/runtime/src/main/java/jsheets/runtime/ConfigModule.java index dab7fb6..031cd14 100644 --- a/runtime/src/main/java/jsheets/runtime/ConfigModule.java +++ b/runtime/src/main/java/jsheets/runtime/ConfigModule.java @@ -10,7 +10,7 @@ import jsheets.config.CombinedConfig; import jsheets.config.Config; import jsheets.config.EnvironmentConfig; -import jsheets.runtime.evaluation.SandboxConfigSource; +import jsheets.runtime.evaluation.EvaluationConfigSource; final class ConfigModule extends AbstractModule { static ConfigModule create() { @@ -26,7 +26,7 @@ private ConfigModule() {} Config createConfig(@Named("environment") Config environment) { var configs = new ArrayList(); configs.add(environment); - configs.add(SandboxConfigSource.fromClassPath().load()); + configs.add(EvaluationConfigSource.fromClassPath().load()); return CombinedConfig.of(configs.toArray(Config[]::new)); } diff --git a/runtime/src/main/java/jsheets/runtime/evaluation/SandboxConfigSource.java b/runtime/src/main/java/jsheets/runtime/evaluation/EvaluationConfigSource.java similarity index 50% rename from runtime/src/main/java/jsheets/runtime/evaluation/SandboxConfigSource.java rename to runtime/src/main/java/jsheets/runtime/evaluation/EvaluationConfigSource.java index b4c0a1d..abf54d4 100644 --- a/runtime/src/main/java/jsheets/runtime/evaluation/SandboxConfigSource.java +++ b/runtime/src/main/java/jsheets/runtime/evaluation/EvaluationConfigSource.java @@ -2,19 +2,27 @@ import java.io.BufferedInputStream; import java.io.IOException; +import java.util.Collection; +import java.util.List; import java.util.Optional; import com.google.common.flogger.FluentLogger; +import com.google.inject.Provides; + +import javax.inject.Named; import jsheets.config.Config; import jsheets.config.RawConfig; +import jsheets.runtime.ServerSetup; /** * Reads the {@code AccessGraph} configuration from the classpath. */ -public final class SandboxConfigSource implements Config.Source { +public final class EvaluationConfigSource implements Config.Source { private static final FluentLogger log = FluentLogger.forEnclosingClass(); + private EvaluationConfigSource() {} + private static final Config.Key disableSandboxKey = Config.Key.ofFlag("evaluation.sandbox.disable"); @@ -29,32 +37,53 @@ public static Config.Key disableSandboxKey() { private static final Config.Key accessGraphKey = Config.Key.ofString("evaluation.sandbox.accessGraph"); + private static final Config.Key defaultImportsKey = + Config.Key.ofString("evaluation.defaultImports"); + public static Config.Key accessGraphKey() { return accessGraphKey; } - public static SandboxConfigSource fromClassPath() { - return new SandboxConfigSource(); + public static EvaluationConfigSource fromClassPath() { + return new EvaluationConfigSource(); } - private SandboxConfigSource() {} + private static final Config.Key virtualMachineOptionsKey = + Config.Key.ofString("evaluation.fork.virtualMachineOptions"); + + static Config.Key virtualMachineOptionsKey() { + return virtualMachineOptionsKey; + } + + private static final String accessGraphFilePath = + "runtime/evaluation/sandbox/accessGraph.txt"; + + private static final String virtualMachineOptionsFilePath = + "runtime/evaluation/fork/virtualMachineOptions.txt"; + + private static final String defaultImportsFilePath = + "runtime/evaluation/defaultImportsKey.txt"; @Override public Config load() { var config = RawConfig.newBuilder(); - readAccessGraphFile() - .ifPresent(accessGraph -> config.with(accessGraphKey, accessGraph)); + readFullFile(accessGraphFilePath).ifPresent(value -> + config.with(accessGraphKey, value) + ); + readFullFile(virtualMachineOptionsFilePath).ifPresent(value -> + config.with(virtualMachineOptionsKey, value) + ); + readFullFile(defaultImportsFilePath).ifPresent(value -> + config.with(defaultImportsKey, value) + ); return config.create(); } - private static final String accessGraphFilePath = - "runtime/evaluation/sandbox/accessGraph.txt"; - - private Optional readAccessGraphFile() { + private static Optional readFullFile(String path) { var resources = Thread.currentThread().getContextClassLoader(); - var file = resources.getResourceAsStream(accessGraphFilePath); + var file = resources.getResourceAsStream(path); if (file == null) { - log.atConfig().log("could not find %s in classpath", accessGraphFilePath); + log.atConfig().log("could not find %s in classpath", path); return Optional.empty(); } try (var input = new BufferedInputStream(file)) { @@ -62,7 +91,7 @@ private Optional readAccessGraphFile() { } catch (IOException failedRead) { log.atWarning() .withCause(failedRead) - .log("failed to read accessGraph.txt"); + .log("failed to read %s", path); } return Optional.empty(); } diff --git a/runtime/src/main/java/jsheets/runtime/evaluation/EvaluationModule.java b/runtime/src/main/java/jsheets/runtime/evaluation/EvaluationModule.java index 0c4e594..7dfdf5e 100644 --- a/runtime/src/main/java/jsheets/runtime/evaluation/EvaluationModule.java +++ b/runtime/src/main/java/jsheets/runtime/evaluation/EvaluationModule.java @@ -4,20 +4,25 @@ import com.google.inject.Provides; import com.google.inject.Singleton; +import javax.inject.Named; import jsheets.evaluation.EvaluationEngine; import jsheets.evaluation.sandbox.access.AccessGraph; import jsheets.evaluation.sandbox.validation.ForbiddenMemberFilter; import jsheets.evaluation.shell.ShellEvaluationEngine; import jsheets.evaluation.shell.environment.ExecutionEnvironment; -import jsheets.evaluation.shell.environment.SandboxedEnvironment; +import jsheets.evaluation.shell.environment.fork.ForkedExecutionEnvironment; +import jsheets.evaluation.shell.environment.sandbox.SandboxClassFileCheck; +import jsheets.evaluation.shell.environment.sandbox.SandboxedEnvironment; import jsheets.evaluation.shell.environment.StandardEnvironment; import jsheets.config.Config; import jsheets.evaluation.shell.execution.SystemBasedExecutionMethodFactory; +import java.util.Collection; import java.util.List; -import static jsheets.runtime.evaluation.SandboxConfigSource.accessGraphKey; -import static jsheets.runtime.evaluation.SandboxConfigSource.disableSandboxKey; +import static jsheets.runtime.evaluation.EvaluationConfigSource.accessGraphKey; +import static jsheets.runtime.evaluation.EvaluationConfigSource.disableSandboxKey; +import static jsheets.runtime.evaluation.EvaluationConfigSource.virtualMachineOptionsKey; public final class EvaluationModule extends AbstractModule { public static EvaluationModule create() { @@ -45,8 +50,18 @@ ExecutionEnvironment executionEnvironment(Config config) { } var accessGraphConfig = accessGraphKey().in(config).require(); var accessGraph = AccessGraph.of(accessGraphConfig.split("\n")); - return SandboxedEnvironment.create( - List.of(ForbiddenMemberFilter.create(accessGraph)) + return ForkedExecutionEnvironment.create( + SandboxClassFileCheck.of( + List.of(ForbiddenMemberFilter.create(accessGraph)) + ), + listVirtualMachineOptions(config) ); } + + Collection listVirtualMachineOptions(Config config) { + return List.of( + virtualMachineOptionsKey().in(config).or("").trim().split("\n") + ); + } + } \ No newline at end of file diff --git a/runtime/src/main/resources/runtime/evaluation/fork/virtualMachineOptions.txt b/runtime/src/main/resources/runtime/evaluation/fork/virtualMachineOptions.txt new file mode 100644 index 0000000..d274d62 --- /dev/null +++ b/runtime/src/main/resources/runtime/evaluation/fork/virtualMachineOptions.txt @@ -0,0 +1,3 @@ +-Xmx8M +-Xms8M +-XX:+UseSerialGC \ No newline at end of file From a6cd80decf4fa90ae5cc68307293ee132981a329 Mon Sep 17 00:00:00 2001 From: Merlin Osayimwen Date: Sat, 16 Oct 2021 19:20:12 +0200 Subject: [PATCH 14/14] Fix forked environment --- .../environment/EmptyClassFileStore.java | 17 +++++++ .../fork/ForkingExecutionControlProvider.java | 46 +++++++++++-------- 2 files changed, 45 insertions(+), 18 deletions(-) create mode 100644 evaluation/src/main/java/jsheets/evaluation/shell/environment/EmptyClassFileStore.java diff --git a/evaluation/src/main/java/jsheets/evaluation/shell/environment/EmptyClassFileStore.java b/evaluation/src/main/java/jsheets/evaluation/shell/environment/EmptyClassFileStore.java new file mode 100644 index 0000000..c794a7d --- /dev/null +++ b/evaluation/src/main/java/jsheets/evaluation/shell/environment/EmptyClassFileStore.java @@ -0,0 +1,17 @@ +package jsheets.evaluation.shell.environment; + +import jdk.jshell.spi.ExecutionControl; + +public final class EmptyClassFileStore implements ClassFileStore { + public static EmptyClassFileStore create() { + return new EmptyClassFileStore(); + } + + private EmptyClassFileStore() {} + + @Override + public void load(ExecutionControl.ClassBytecodes[] bytecodes) {} + + @Override + public void redefine(ExecutionControl.ClassBytecodes[] bytecodes) {} +} \ No newline at end of file diff --git a/evaluation/src/main/java/jsheets/evaluation/shell/environment/fork/ForkingExecutionControlProvider.java b/evaluation/src/main/java/jsheets/evaluation/shell/environment/fork/ForkingExecutionControlProvider.java index 74ca3cb..dd4adeb 100644 --- a/evaluation/src/main/java/jsheets/evaluation/shell/environment/fork/ForkingExecutionControlProvider.java +++ b/evaluation/src/main/java/jsheets/evaluation/shell/environment/fork/ForkingExecutionControlProvider.java @@ -6,14 +6,17 @@ import java.io.ObjectOutput; import java.io.OutputStream; import java.net.InetAddress; -import java.net.InetSocketAddress; import java.net.ServerSocket; import java.time.Duration; +import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.function.BiFunction; +import java.util.function.Consumer; import com.sun.jdi.VirtualMachine; import jdk.jshell.execution.JdiInitiator; @@ -23,6 +26,7 @@ import jdk.jshell.spi.ExecutionControlProvider; import jdk.jshell.spi.ExecutionEnv; import jsheets.evaluation.shell.environment.ClassFileStore; +import jsheets.evaluation.shell.environment.EmptyClassFileStore; import static jdk.jshell.execution.Util.remoteInputOutput; @@ -31,6 +35,10 @@ public final class ForkingExecutionControlProvider private static final Duration defaultTimeout = Duration.ofMillis(3000); + public static ForkingExecutionControlProvider create() { + return create(List.of(), EmptyClassFileStore.create()); + } + public static ForkingExecutionControlProvider create( Collection rawVirtualMachineOptions, ClassFileStore classFileStore @@ -91,10 +99,10 @@ private Box initiate(int port) { /* port */ port, /* options */ rawVirtualMachineOptions, /* remoteAgentClassName */ remoteAgentClassName, - /* controlledLaunch */ true, - /* host */ host, + /* controlledLaunch */ false, + /* host */ "", /* timeout */ (int) timeout.toMillis(), - /* connectorOptions*/ Map.of() + /* connectorOptions*/ Collections.emptyMap() ); return new Box( initiator.vm(), @@ -118,6 +126,7 @@ private ExecutionControl accept( ExecutionEnv environment, Box box ) throws IOException { + var hooks = registerCloseHooks(box.machine()); var socket = listener.accept(); var output = socket.getOutputStream(); return remoteInputOutput( @@ -125,13 +134,24 @@ private ExecutionControl accept( output, createOutputs(environment), createInputs(environment), - createControl(box, environment) + createControl(box, environment, hooks) ); } + private Collection> registerCloseHooks(VirtualMachine machine) { + var hooks = new CopyOnWriteArrayList>(); + Util.detectJdiExitEvent(machine, event -> { + for (var hook : hooks) { + hook.accept(event); + } + }); + return hooks; + } + private BiFunction createControl( Box box, - ExecutionEnv environment + ExecutionEnv environment, + Collection> hooks ) { return (input, output) -> { var control = new ForkedExecutionControl( @@ -142,7 +162,8 @@ private BiFunction createControl( remoteAgentClassName, classFileStore ); - registerCloseHooks(box.machine(), environment, control); + hooks.add(event -> environment.closeDown()); + hooks.add(event -> control.disposeMachine()); return control; }; } @@ -157,15 +178,4 @@ private Map createOutputs(ExecutionEnv environment) { private Map createInputs(ExecutionEnv environment) { return Map.of("in", environment.userIn()); } - - private void registerCloseHooks( - VirtualMachine machine, - ExecutionEnv environment, - ForkedExecutionControl control - ) { - Util.detectJdiExitEvent(machine, event -> { - environment.closeDown(); - control.disposeMachine(); - }); - } }