Skip to content

Enhance GSON Converters, hash data only once, cache ULID generator class #137

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Open

Enhance GSON Converters, hash data only once, cache ULID generator class #137

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
c309ce1
include dependencies inside package jar
Override-6 May 15, 2024
b409668
hash data only once
Override-6 May 27, 2024
6d73d3a
cache ULID generator in static memory
Override-6 May 27, 2024
bc8e871
optimize JSON serialization
Override-6 May 15, 2024
b5374cb
apply suggestions
Override-6 Jun 7, 2024
01cd65f
fix NPE in AnnotationConverter, fix compilation errors in tests
Override-6 Jun 11, 2024
097891f
add AnnotationListConverter
Override-6 Jun 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
.idea/
.vscode/
alvarium-sdk-java.iml
sbom-tool

# Compiled class file
*.class
Expand Down
95 changes: 60 additions & 35 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,39 @@

<name>alvarium-sdk</name>
<url>https://alvarium.org/</url>

<properties>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
</properties>

<build>
<plugins>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.6.0</version>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>

<dependencies>

<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
Expand All @@ -30,55 +55,55 @@
<version>4.11</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.google.crypto.tink</groupId>
<artifactId>tink</artifactId>
<version>1.6.1</version>
</dependency>

<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.8.8</version>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.8.8</version>
</dependency>

<dependency>
<groupId>de.huxhorn.sulky</groupId>
<artifactId>de.huxhorn.sulky.ulid</artifactId>
<version>8.2.0</version>
</dependency>

<dependency>
<groupId>de.huxhorn.sulky</groupId>
<artifactId>de.huxhorn.sulky.ulid</artifactId>
<version>8.2.0</version>
<groupId>org.eclipse.paho</groupId>
<artifactId>org.eclipse.paho.client.mqttv3</artifactId>
<version>1.2.5</version>
</dependency>

<dependency>
<groupId>org.eclipse.paho</groupId>
<artifactId>org.eclipse.paho.client.mqttv3</artifactId>
<version>1.2.5</version>
<groupId>io.pravega</groupId>
<artifactId>pravega-client</artifactId>
<version>0.10.0</version>
</dependency>

<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>2.21.0</version>
</dependency>

<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.21.0</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>io.pravega</groupId>
<artifactId>pravega-client</artifactId>
<version>0.10.0</version>
<groupId>org.spdx</groupId>
<artifactId>spdx-jackson-store</artifactId>
<version>1.1.9.1</version>
</dependency>

<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>2.21.0</version>
</dependency>

<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.21.0</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.spdx</groupId>
<artifactId>spdx-jackson-store</artifactId>
<version>1.1.9.1</version>
</dependency>

</dependencies>
</project>
</project>
79 changes: 46 additions & 33 deletions src/main/java/com/alvarium/DefaultSdk.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,25 +14,27 @@
*******************************************************************************/
package com.alvarium;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;

import com.alvarium.annotators.Annotator;
import com.alvarium.annotators.AnnotatorConfig;
import com.alvarium.annotators.AnnotatorException;
import com.alvarium.annotators.AnnotatorFactory;
import com.alvarium.contracts.Annotation;
import com.alvarium.contracts.AnnotationList;
import com.alvarium.contracts.AnnotationType;
import com.alvarium.hash.HashProviderFactory;
import com.alvarium.hash.HashTypeException;
import com.alvarium.streams.StreamException;
import com.alvarium.streams.StreamProvider;
import com.alvarium.streams.StreamProviderFactory;
import com.alvarium.utils.ImmutablePropertyBag;
import com.alvarium.utils.PropertyBag;

import org.apache.logging.log4j.Logger;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;

Comment on lines -17 to +36
Copy link
Contributor

@Ali-Amin Ali-Amin Jun 6, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please refer to our style guide which contains a section for imports and the correct ordering and grouping. It's a combination of industry styling guides used by Google and Twitter.

I understand these changes are probably due to a formatter, but what maintainers here usually do is create configuration on the editor of choice prior to making changes that reflect what is in that style guide.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No problem, why not trying to use a maven plugin to help apply the code style policy ?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't see why not, we can look into it.


public class DefaultSdk implements Sdk {
private final Annotator[] annotators;
private final SdkInfo config;
Expand All @@ -51,20 +53,20 @@ public DefaultSdk(Annotator[] annotators, SdkInfo config, Logger logger) throws
this.logger.debug("stream provider connected successfully.");
}

public void create(PropertyBag properties, byte[] data) throws AnnotatorException,
StreamException {
public void create(PropertyBag properties, byte[] data) throws AnnotatorException,
StreamException, HashTypeException {
final List<Annotation> annotations = this.createAnnotations(properties, data);
this.publishAnnotations(SdkAction.CREATE, annotations);
this.logger.debug("data annotated and published successfully.");
}
public void create(byte[] data) throws AnnotatorException, StreamException {
final PropertyBag properties = new ImmutablePropertyBag(new HashMap<String, Object>());

public void create(byte[] data) throws AnnotatorException, StreamException, HashTypeException {
final PropertyBag properties = new ImmutablePropertyBag(new HashMap<>());
this.create(properties, data);
}

public void mutate(PropertyBag properties, byte[] oldData, byte[] newData) throws
AnnotatorException, StreamException {
public void mutate(PropertyBag properties, byte[] oldData, byte[] newData) throws
AnnotatorException, StreamException, HashTypeException {
final List<Annotation> annotations = new ArrayList<Annotation>();

// source annotate the old data
Expand All @@ -74,14 +76,19 @@ public void mutate(PropertyBag properties, byte[] oldData, byte[] newData) throw
this.config,
this.logger
);
final Annotation sourceAnnotation = sourceAnnotator.execute(properties, oldData);

String key;
var hasher = new HashProviderFactory().getProvider(config.getHash().getType());
key = hasher.derive(oldData);

final Annotation sourceAnnotation = sourceAnnotator.execute(properties, oldData, key);
annotations.add(sourceAnnotation);

// Add annotations for new data
for (Annotation annotation: this.createAnnotations(properties, newData)) {
for (Annotation annotation : this.createAnnotations(properties, newData)) {
// TLS is ignored in mutate to prevent needless penalization
// See https://github.com/project-alvarium/alvarium-sdk-go/issues/19
if(annotation.getKind() != AnnotationType.TLS) {
if (annotation.getKind() != AnnotationType.TLS) {
annotations.add(annotation);
}
}
Expand All @@ -91,32 +98,32 @@ public void mutate(PropertyBag properties, byte[] oldData, byte[] newData) throw
this.logger.debug("data annotated and published successfully.");
}

public void mutate(byte[] oldData, byte[] newData) throws AnnotatorException, StreamException {
final PropertyBag properties = new ImmutablePropertyBag(new HashMap<String, Object>());
public void mutate(byte[] oldData, byte[] newData) throws AnnotatorException, StreamException, HashTypeException {
final PropertyBag properties = new ImmutablePropertyBag(new HashMap<>());
this.mutate(properties, oldData, newData);
}

public void transit(PropertyBag properties, byte[] data) throws AnnotatorException,
StreamException {
StreamException, HashTypeException {
final List<Annotation> annotations = this.createAnnotations(properties, data);
this.publishAnnotations(SdkAction.TRANSIT, annotations);
this.logger.debug("data annotated and published successfully.");
}

public void transit(byte[] data) throws AnnotatorException, StreamException {
public void transit(byte[] data) throws AnnotatorException, StreamException, HashTypeException {
final PropertyBag properties = new ImmutablePropertyBag(new HashMap<String, Object>());
this.transit(properties, data);
}

public void publish(PropertyBag properties, byte[] data) throws AnnotatorException,
StreamException {
StreamException, HashTypeException {
final List<Annotation> annotations = this.createAnnotations(properties, data);
this.publishAnnotations(SdkAction.PUBLISH, annotations);
this.logger.debug("data annotated and published successfully.");
}
public void publish(byte[] data) throws AnnotatorException, StreamException {
final PropertyBag properties = new ImmutablePropertyBag(new HashMap<String, Object>());

public void publish(byte[] data) throws AnnotatorException, StreamException, HashTypeException {
final PropertyBag properties = new ImmutablePropertyBag(new HashMap<>());
this.publish(properties, data);
}

Expand All @@ -127,39 +134,45 @@ public void close() throws StreamException {

/**
* Executes all the specified annotators and returns a list of all the created annotations
*
* @param properties
* @param data
* @return
* @throws AnnotatorException
*/
private List<Annotation> createAnnotations(PropertyBag properties, byte[] data)
throws AnnotatorException {
final List<Annotation> annotations = new ArrayList<Annotation>();
private List<Annotation> createAnnotations(PropertyBag properties, byte[] data)
throws AnnotatorException, HashTypeException {
final List<Annotation> annotations = new ArrayList<>();

String key;
var hasher = new HashProviderFactory().getProvider(config.getHash().getType());
key = hasher.derive(data);

// Annotate incoming data
for (Annotator annotator: this.annotators) {
final Annotation annotation = annotator.execute(properties, data);
for (Annotator annotator : this.annotators) {
final Annotation annotation = annotator.execute(properties, data, key);
annotations.add(annotation);
}

return annotations;
}

/**
* Wraps the annotation list with a publish wrapper that specifies the SDK action and the
* Wraps the annotation list with a publish wrapper that specifies the SDK action and the
* content type
*
* @param action
* @param annotations
* @throws StreamException
*/
private void publishAnnotations(SdkAction action, List<Annotation> annotations)
private void publishAnnotations(SdkAction action, List<Annotation> annotations)
throws StreamException {
final AnnotationList annotationList = new AnnotationList(annotations);

// publish list of annotations to the StreamProvider
final PublishWrapper wrapper = new PublishWrapper(
action,
annotationList.getClass().getName(),
action,
annotationList.getClass().getName(),
annotationList
);
this.stream.publish(wrapper);
Expand Down
37 changes: 6 additions & 31 deletions src/main/java/com/alvarium/PublishWrapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,9 @@
*******************************************************************************/
package com.alvarium;

import java.io.Serializable;
import java.util.Base64;
import com.alvarium.serializers.Serialization;

import com.alvarium.contracts.Annotation;
import com.alvarium.serializers.AnnotationConverter;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonElement;
import java.io.Serializable;

/**
* A java bean that encapsulates the content sent through the stream providers
Expand All @@ -32,6 +27,7 @@ public class PublishWrapper implements Serializable {
private final String messageType;
private final Object content;


public PublishWrapper(SdkAction action, String messageType, Object content) {
this.action = action;
this.messageType = messageType;
Expand All @@ -50,33 +46,12 @@ public Object getContent() {
return content;
}


/**
* The content field in the returned JSON will be Base64 string encoded
* The content field in the returned JSON will be Base64 string encoded
* @return String representation of the PublishWrapper JSON
*/
public String toJson() {
Gson gson = new GsonBuilder()
.registerTypeAdapter(Annotation.class, new AnnotationConverter())
.disableHtmlEscaping()
.create();

// Change the content field to a base64 encoded string before serializing to json
final JsonElement decodedContent = gson.toJsonTree(this.content);
final String encodedContent;

// `toString()` will work if the content is a primitive type, but will add additional
// quotes (e.g. "foo" will be "\"foo\"") but `getAsString()` will produce correct behavior but
// using `getAsString()` on a non-primitive type will throw an exception.
// This condition ensures that the correct method is called on the correct type
if (decodedContent.isJsonPrimitive()) {
encodedContent = Base64.getEncoder().encodeToString(decodedContent.getAsString().getBytes());
} else {
encodedContent = Base64.getEncoder().encodeToString(decodedContent.toString().getBytes());
}

// new publish wrapper returned as JSON string with encoded content
// to prevent setting the object content value
final PublishWrapper wrapper = new PublishWrapper(action, messageType, encodedContent);
return gson.toJson(wrapper);
return Serialization.toJson(this);
}
}
Loading