java-sheets · ehenoma · Oct 18, 2021 · Oct 18, 2021 · Oct 18, 2021 · Oct 18, 2021
diff --git a/build.gradle b/build.gradle
@@ -14,4 +14,6 @@ ext {
   consuleClientVersion = '1.4.5'
   javaxAnnotationVersion = '1.3.2'
   recordBuilderVersion = '28'
+  guavaVersion = '31.0.1-jre'
+  bucket4jVersion = '6.3.0'
 }
diff --git a/common/build.gradle b/common/build.gradle
@@ -0,0 +1,25 @@
+plugins {
+  id 'java'
+}
+
+group 'dev.jsheets'
+version '0.1.0'
+
+sourceCompatibility = 16
+targetCompatibility = 16
+
+repositories {
+  mavenCentral()
+}
+
+dependencies {
+  testImplementation "org.junit.jupiter:junit-jupiter-api:$junitPlatformVersion"
+  testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:$junitPlatformVersion"
+  implementation "com.google.guava:guava:$guavaVersion"
+  compileOnly "io.soabase.record-builder:record-builder-core:$recordBuilderVersion"
+  annotationProcessor "io.soabase.record-builder:record-builder-processor:$recordBuilderVersion"
+}
+
+test {
+  useJUnitPlatform()
+}
diff --git a/...c/main/java/jsheets/config/CamelCase.java → ...c/main/java/jsheets/config/CamelCase.java b/...c/main/java/jsheets/config/CamelCase.java → ...c/main/java/jsheets/config/CamelCase.java
diff --git a/...n/java/jsheets/config/CombinedConfig.java → ...n/java/jsheets/config/CombinedConfig.java b/...n/java/jsheets/config/CombinedConfig.java → ...n/java/jsheets/config/CombinedConfig.java
diff --git a/.../src/main/java/jsheets/config/Config.java → .../src/main/java/jsheets/config/Config.java b/.../src/main/java/jsheets/config/Config.java → .../src/main/java/jsheets/config/Config.java
diff --git a/common/src/main/java/jsheets/config/Configs.java b/common/src/main/java/jsheets/config/Configs.java
@@ -0,0 +1,13 @@
+package jsheets.config;
+
+public final class Configs {
+  private Configs() {}
+
+  public static Config loadAll(Config.Source... sources) {
+    var configs = new Config[sources.length];
+    for (int source = 0; source < sources.length; source++) {
+      configs[source] = sources[source].load();
+    }
+    return CombinedConfig.of(configs);
+  }
+}
diff --git a/...ava/jsheets/config/EnvironmentConfig.java → ...ava/jsheets/config/EnvironmentConfig.java b/...ava/jsheets/config/EnvironmentConfig.java → ...ava/jsheets/config/EnvironmentConfig.java
diff --git a/common/src/main/java/jsheets/config/FileConfigSource.java b/common/src/main/java/jsheets/config/FileConfigSource.java
@@ -0,0 +1,62 @@
+package jsheets.config;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Map;
+import java.util.Optional;
+
+import io.soabase.recordbuilder.core.RecordBuilder;
+
+@RecordBuilder
+public record FileConfigSource(
+  Config.Key<?> key,
+  Path relativePath,
+  Path workingDirectory
+) implements Config.Source {
+
+  public static FileConfigSource inCurrentWorkingDirectory(
+    Config.Key<?> key,
+    Path relativePath
+  ) {
+    var workingDirectory = Path.of(System.getProperty("user.dir"));
+    return new FileConfigSource(key, relativePath, workingDirectory);
+  }
+
+  @Override
+  public Config load() {
+    return loadContent()
+      .map(content -> RawConfig.of(Map.of(key.toString(), content)))
+      .orElseGet(() -> RawConfig.of(Map.of()));
+  }
+
+  private Optional<String> loadContent() {
+    return loadFromSystem().or(this::loadFromClassPath);
+  }
+
+  private Optional<String> loadFromSystem() {
+    var fullPath = workingDirectory.resolve(relativePath);
+    try {
+      return Optional.of(
+        Files.readString(fullPath, StandardCharsets.UTF_8)
+      );
+    } catch (IOException failedRead) {
+      return Optional.empty();
+    }
+  }
+
+  private Optional<String> loadFromClassPath() {
+    var resources = Thread.currentThread().getContextClassLoader();
+    var file = resources.getResourceAsStream(relativePath.toString());
+    if (file == null) {
+      return Optional.empty();
+    }
+    try (var input = new BufferedInputStream(file)) {
+      return Optional.of(new String(input.readAllBytes()));
+    } catch (IOException failedRead) {
+      return Optional.empty();
+    }
+  }
+}
diff --git a/...ain/java/jsheets/config/MissingField.java → ...ain/java/jsheets/config/MissingField.java b/...ain/java/jsheets/config/MissingField.java → ...ain/java/jsheets/config/MissingField.java
diff --git a/...c/main/java/jsheets/config/RawConfig.java → ...c/main/java/jsheets/config/RawConfig.java b/...c/main/java/jsheets/config/RawConfig.java → ...c/main/java/jsheets/config/RawConfig.java
diff --git a/...in/java/jsheets/config/ResolvedField.java → ...in/java/jsheets/config/ResolvedField.java b/...in/java/jsheets/config/ResolvedField.java → ...in/java/jsheets/config/ResolvedField.java
diff --git a/common/src/main/java/jsheets/event/EventSink.java b/common/src/main/java/jsheets/event/EventSink.java
@@ -0,0 +1,24 @@
+package jsheets.event;
+
+import java.util.function.Supplier;
+
+public interface EventSink {
+  void post(Object event);
+
+  /**
+   * Only creates the event to post if the sink is enabled.
+   * <p>
+   * This method is preferred if creating an event is associated with some
+   * overhead. It only runs the {@code eventFactory}, if there is a chance that
+   * it is subscribed.
+   *
+   * @param eventFactory Lazily creates the event
+   */
+  default void postIfEnabled(Supplier<Object> eventFactory) {
+    post(eventFactory.get());
+  }
+
+  static EventSink ignore() {
+    return event -> {};
+  }
+}
diff --git a/common/src/main/java/jsheets/event/GuavaEventSink.java b/common/src/main/java/jsheets/event/GuavaEventSink.java
@@ -0,0 +1,28 @@
+package jsheets.event;
+
+import java.util.Objects;
+
+import com.google.common.eventbus.EventBus;
+
+public final class GuavaEventSink implements EventSink {
+  public static EventSink forBus(EventBus bus) {
+    Objects.requireNonNull(bus, "bus");
+    return new GuavaEventSink(bus);
+  }
+
+  private final EventBus bus;
+
+  private GuavaEventSink(EventBus bus) {
+    this.bus = bus;
+  }
+
+  @Override
+  public void post(Object event) {
+    bus.post(event);
+  }
+
+  @Override
+  public String toString() {
+    return "GuavaEventSink(bus=%s)".formatted(bus);
+  }
+}
diff --git a/common/src/main/java/jsheets/event/LabeledEvent.java b/common/src/main/java/jsheets/event/LabeledEvent.java
@@ -0,0 +1,7 @@
+package jsheets.event;
+
+import java.util.Map;
+
+public interface LabeledEvent {
+  Map<String, Object> labels();
+}
diff --git a/...st/java/jsheets/config/CamelCaseTest.java → ...st/java/jsheets/config/CamelCaseTest.java b/...st/java/jsheets/config/CamelCaseTest.java → ...st/java/jsheets/config/CamelCaseTest.java
diff --git a/...jsheets/config/EnvironmentConfigTest.java → ...jsheets/config/EnvironmentConfigTest.java b/...jsheets/config/EnvironmentConfigTest.java → ...jsheets/config/EnvironmentConfigTest.java
diff --git a/deploy/minimal/README.md → deploy/compose/README.md b/deploy/minimal/README.md → deploy/compose/README.md
diff --git a/deploy/compose/deploy.sh b/deploy/compose/deploy.sh
@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+docker-compose -f monitoring.yml -f main.yml "$@"
diff --git a/deploy/minimal/docker-compose.yml → deploy/compose/main.yml b/deploy/minimal/docker-compose.yml → deploy/compose/main.yml
@@ -3,12 +3,19 @@ services:
   server:
     image: ehenoma/jsheets:latest
     restart: always
+    build:
+      context: ../../
+      dockerfile: ./server/deploy/Dockerfile
     environment:
       JSHEETS_SERVER_PORT: 8080
-      JSHEETS_MONGODB_URI: mongodb://root:root@document-store/jsheets
+      JSHEETS_MONGODB_URI: mongodb://root:root@document-store/jsheets?authSource=admin
       JSHEETS_ZOOKEEPER_CONNECTION_STRING: zookeeper:2181
+      JSHEETS_SERVER_RATE_LIMIT_CAPACITY: 100
+      JSHEETS_SERVER_RATE_LIMIT_REFILL_PER_SECOND: 50
     ports:
       - "8080:8080"
+    depends_on:
+      - document-store
     networks:
       - document-store
       - zookeeper
@@ -38,14 +45,19 @@ services:
     restart: always
     container_name: runtime
     hostname: runtime
+    build:
+      context: ../../
+      dockerfile: ./runtime/deploy/Dockerfile
     environment:
       JSHEETS_RUNTIME_SERVER_PORT: 8080
       JSHEETS_RUNTIME_ZOOKEEPER_CONNECTION_STRING: zookeeper:2181
       JSHEETS_RUNTIME_SERVICE_ADVERTISED_HOST: runtime:8080
+    depends_on:
+      - zookeeper
     networks:
       - zookeeper
       - runtime
 networks:
   document-store:
   zookeeper:
-  runtime:
+  runtime:
diff --git a/deploy/compose/monitoring.yml b/deploy/compose/monitoring.yml
@@ -0,0 +1,35 @@
+version: '3.7'
+services:
+  influxdb:
+    image: influxdb:2.0
+    hostname: influxdb
+    ports:
+      - '8086:8086'
+    volumes:
+      - influxdb-storage:/var/lib/influxdb
+    environment:
+      DOCKER_INFLUXDB_DB: jsheets
+      DOCKER_INFLUXDB_INIT_ORG: jsheets
+      DOCKER_INFLUXDB_INIT_BUCKET: jsheets
+      DOCKER_INFLUXDB_INIT_USERNAME: ${INFLUXDB_USERNAME}
+      DOCKER_INFLUXDB_INIT_PASSWORD: ${INFLUXDB_PASSWORD}
+      DOCKER_INFLUXDB_INIT_ADMIN_TOKEN: ${INFLUXDB_TOKEN}
+    networks:
+      - influxdb
+  chronograf:
+    image: chronograf:latest
+    ports:
+      - '127.0.0.1:8888:8888'
+    volumes:
+      - chronograf-storage:/var/lib/chronograf
+    depends_on:
+      - influxdb
+    environment:
+      INFLUXDB_URL: http://influxdb:8086
+      INFLUXDB_USERNAME: ${INFLUXDB_USERNAME}
+      INFLUXDB_PASSWORD: ${INFLUXDB_PASSWORD}
+volumes:
+  influxdb-storage:
+  chronograf-storage:
+networks:
+ influxdb:
diff --git a/evaluation/README.md b/evaluation/README.md
@@ -0,0 +1,15 @@
+## Events
+When building the `EvaluationEngine` you can pass an `EventSink`, it will then
+post events type `EvaluationEvent`. Those events can be used to record
+metrics and run custom code with pretty loose coupling.
+
+If no `EventSink` is configured, the engine will typically not attempt to
+create any events, thus no overhead is produced.
+
+Following events are currently posted:
+
+| Name  | Posted by | When   |
+|-------|-----------|--------|
+| `EvaluationStartEvent` | `EvaluationEngine` | An evaluation begins
+| `EvaluationStopEvent` | `EvaluationEngine` | An evaluation completes or fails
+| `BoxLifecycleEvent` | `ForkedExecutionEnvironment` | A JVM is starting/ready/stopping |
diff --git a/evaluation/build.gradle b/evaluation/build.gradle
@@ -18,12 +18,13 @@ ext {
 
 dependencies {
   implementation project(':protocol')
+  implementation project(':common')
   implementation "org.ow2.asm:asm:$asmVersion"
   implementation "com.google.protobuf:protobuf-java-util:$protobufJavaVersion"
   implementation "com.google.flogger:flogger:$floggerVersion"
+  testRuntimeOnly "com.google.flogger:flogger-slf4j-backend:$floggerVersion"
   testImplementation "org.junit.jupiter:junit-jupiter-api:$junitPlatformVersion"
   testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:$junitPlatformVersion"
-  testRuntimeOnly "com.google.flogger:flogger-slf4j-backend:$floggerVersion"
   compileOnly "io.soabase.record-builder:record-builder-core:$recordBuilderVersion"
   annotationProcessor "io.soabase.record-builder:record-builder-processor:$recordBuilderVersion"
 }

diff --git a/evaluation/src/main/java/jsheets/evaluation/EvaluationEvent.java b/evaluation/src/main/java/jsheets/evaluation/EvaluationEvent.java
@@ -0,0 +1,11 @@
+package jsheets.evaluation;
+
+import jsheets.event.LabeledEvent;
+
+/**
+ * The evaluation engine produces events during preprocessing and evaluation
+ * of snippets that can be used to create metrics and trigger custom code.
+ */
+public interface EvaluationEvent extends LabeledEvent {
+  String snippetId();
+}
diff --git a/evaluation/src/main/java/jsheets/evaluation/EvaluationStartEvent.java b/evaluation/src/main/java/jsheets/evaluation/EvaluationStartEvent.java
@@ -0,0 +1,11 @@
+package jsheets.evaluation;
+
+import java.util.Map;
+
+import io.soabase.recordbuilder.core.RecordBuilder;
+
+@RecordBuilder
+public record EvaluationStartEvent(
+  String snippetId,
+  Map<String, Object> labels
+) implements EvaluationEvent {}
diff --git a/evaluation/src/main/java/jsheets/evaluation/EvaluationStopEvent.java b/evaluation/src/main/java/jsheets/evaluation/EvaluationStopEvent.java
@@ -0,0 +1,34 @@
+package jsheets.evaluation;
+
+import java.time.Duration;
+import java.util.Map;
+
+import io.soabase.recordbuilder.core.RecordBuilder;
+
+@RecordBuilder
+public record EvaluationStopEvent(
+  String snippetId,
+  Duration duration,
+  Status status,
+  Map<String, Object> labels
+) implements EvaluationEvent {
+
+  public boolean hasFailed() {
+    return status.equals(Status.Failed);
+  }
+
+  public enum Status {
+    /**
+     * The evaluation was successful, there were no errors in the user code.
+     */
+    CompletedSuccessfully,
+    /**
+     * The evaluation could be run but there were errors in the user code.
+     */
+    CompletedWithErrors,
+    /**
+     * The evaluation failed, this must not indicate that user code is invalid.
+     */
+    Failed
+  }
+}
diff --git a/evaluation/src/main/java/jsheets/evaluation/sandbox/ViolationEvent.java b/evaluation/src/main/java/jsheets/evaluation/sandbox/ViolationEvent.java
@@ -0,0 +1,16 @@
+package jsheets.evaluation.sandbox;
+
+import java.util.Collection;
+import java.util.Map;
+
+import io.soabase.recordbuilder.core.RecordBuilder;
+import jsheets.evaluation.EvaluationEvent;
+import jsheets.evaluation.sandbox.validation.Analysis;
+
+@RecordBuilder
+public record ViolationEvent(
+  String snippetId,
+  String componentId,
+  Collection<Analysis.Violation> violations,
+  Map<String, Object> labels
+) implements EvaluationEvent {}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		#!/usr/bin/env bash
		docker-compose -f monitoring.yml -f main.yml "$@"