Pebble template (#75)

* get pending/schedule/processing message count

* Counter for priority based queue

* Pebble template #74

* wip

* pebble template change

* pebble template test fix

* nonce issue

* 2.6.1

Author: sonus21

CHANGELOG.md

@@ -1,5 +1,12 @@
 # [Rqueue] New and Notable Changes
 
+## [2.6.1] - 1-Marc-2021
+
+### Fixes
+
+* Graph not rendering in firefox due to unsafe csp rule
+* Crash in dashboard due to Twig template, changed it to Pebble template
+
 ## [2.6.0] - 22-Feb-2021
 
 Message counts api
@@ -198,3 +205,5 @@ Fixes:
 [2.5.0]: https://repo1.maven.org/maven2/com/github/sonus21/rqueue-core/2.5.0-RELEASE
 
 [2.6.0]: https://repo1.maven.org/maven2/com/github/sonus21/rqueue-core/2.6.0-RELEASE
+
+[2.6.1]: https://repo1.maven.org/maven2/com/github/sonus21/rqueue-core/2.6.1-RELEASE

README.md

@@ -53,14 +53,14 @@ services code is in Spring.
 * Add dependency
   * Gradle
     ```groovy
-        implementation 'com.github.sonus21:rqueue-spring-boot-starter:2.6.0-RELEASE'
+        implementation 'com.github.sonus21:rqueue-spring-boot-starter:2.6.1-RELEASE'
     ```
   * Maven
     ```xml
      <dependency>
         <groupId>com.github.sonus21</groupId>
         <artifactId>rqueue-spring-boot-starter</artifactId>
-        <version>2.6.0-RELEASE</version>
+        <version>2.6.1-RELEASE</version>
     </dependency>
     ```
 
@@ -71,14 +71,14 @@ services code is in Spring.
 * Add Dependency
   * Gradle
     ```groovy
-        implementation 'com.github.sonus21:rqueue-spring:2.6.0-RELEASE'
+        implementation 'com.github.sonus21:rqueue-spring:2.6.1-RELEASE'
     ```
   * Maven
     ```xml
      <dependency>
        <groupId>com.github.sonus21</groupId>
        <artifactId>rqueue-spring</artifactId>
-       <version>2.6.0-RELEASE</version>
+       <version>2.6.1-RELEASE</version>
      </dependency>
     ```
 

build.gradle

@@ -53,7 +53,7 @@ ext {
 
     // server
     javaxServletVersion = '4.0.1'
-    twigVersion = '5.87.0.RELEASE'
+    pebbleVersion = '3.1.5'
 
     // database
     lettuceVersion = '5.2.0.RELEASE'
@@ -63,22 +63,31 @@ ext {
 
     // other dependencies
     javaxValidationApiVersion = '2.0.1.Final'
+    
+    guavaVersion = '30.1-jre'
 
 }
 
 
 subprojects {
     group = 'com.github.sonus21'
-    version = '2.6.0-RELEASE'
+    version = '2.6.1-RELEASE'
 
     dependencies {
         // https://mvnrepository.com/artifact/org.springframework/spring-messaging
         compile group: 'org.springframework', name: 'spring-messaging', version: "${springVersion}"
         // https://mvnrepository.com/artifact/org.springframework.data/spring-data-redis
         compile group: 'org.springframework.data', name: 'spring-data-redis', version: "${springDataVersion}"
 
+        // https://mvnrepository.com/artifact/org.apache.commons/commons-lang3
+        compile group: 'org.apache.commons', name: 'commons-lang3', version: "${lang3Version}"
+
+        // https://mvnrepository.com/artifact/com.google.guava/guava
+        compile group: 'com.google.guava', name: 'guava', version: "${guavaVersion}"
+
         compileOnly "org.projectlombok:lombok:${lombokVersion}"
         annotationProcessor "org.projectlombok:lombok:${lombokVersion}"
+
         testCompile "org.projectlombok:lombok:${lombokVersion}"
         testAnnotationProcessor "org.projectlombok:lombok:${lombokVersion}"
 
@@ -93,8 +102,6 @@ subprojects {
         testCompile group: 'org.mockito', name: 'mockito-junit-jupiter', version: "${mockitoVersion}"
         // https://mvnrepository.com/artifact/org.hamcrest/hamcrest
         testCompile group: 'org.hamcrest', name: 'hamcrest', version: "${hamcrestVersion}"
-        // https://mvnrepository.com/artifact/org.apache.commons/commons-lang3
-        testCompile group: 'org.apache.commons', name: 'commons-lang3', version: "${lang3Version}"
         testCompile("org.springframework.boot:spring-boot-test:${springBootVersion}")
         testCompile group: 'org.junit-pioneer', name: 'junit-pioneer', version: '0.9.0'
 

gradle/code-signing.gradle

@@ -61,8 +61,8 @@ publishing {
     repositories {
         repositories {
             maven {
-                def releasesRepoUrl = "https://oss.sonatype.org/service/local/staging/deploy/maven2"
-                def snapshotsRepoUrl = "https://oss.sonatype.org/content/repositories/snapshots"
+                def releasesRepoUrl = "https://s01.oss.sonatype.org/service/local/staging/deploy/maven2"
+                def snapshotsRepoUrl = "https://s01.oss.sonatype.org/content/repositories/snapshots"
                 url = version.endsWith('SNAPSHOT') ? snapshotsRepoUrl : releasesRepoUrl
                 credentials {
                     username sonatypeUsername

rqueue-core/build.gradle

@@ -15,8 +15,7 @@ dependencies {
     compile group: 'javax.servlet', name: 'javax.servlet-api', version: "${javaxServletVersion}"
     // https://mvnrepository.com/artifact/org.springframework/spring-webmvc
     compile group: 'org.springframework', name: 'spring-webmvc', version: "${springVersion}"
-    // https://mvnrepository.com/artifact/org.jtwig/jtwig-spring
-    compile group: 'org.jtwig', name: 'jtwig-spring', version: "${twigVersion}"
+    compile "io.pebbletemplates:pebble-spring5:${pebbleVersion}"
     compile group: 'io.seruco.encoding', name: 'base62', version: "0.1.3"
     // https://mvnrepository.com/artifact/org.apache.commons/commons-collections4
     compile group: 'org.apache.commons', name: 'commons-collections4', version: '4.4'

rqueue-core/src/main/java/com/github/sonus21/rqueue/annotation/RqueueListener.java

@@ -171,13 +171,16 @@
    * weighted a round robin approach is used, and weight is followed.
    *
    * @return the priority for this listener.
+   * @see #priorityGroup()
    */
   String priority() default "";
 
   /**
-   * Priority group for this listener. More than one priority group can be configured in an
-   * application. Priority groups are useful when inside a group some queue(s) have higher priority
-   * then the queue(s).
+   * Priority group for this listener.
+   *
+   * <p>More than one priority group can be configured in an application. Priority groups are
+   * useful
+   * when inside a group some queue(s) have higher priority than the other queue(s).
    *
    * @return priority group name.
    */

rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RqueueListenerBaseConfig.java

@@ -30,12 +30,11 @@
 import com.github.sonus21.rqueue.dao.impl.RqueueStringDaoImpl;
 import com.github.sonus21.rqueue.metrics.RqueueQueueMetrics;
 import com.github.sonus21.rqueue.utils.RedisUtils;
-import com.github.sonus21.rqueue.web.view.DateTimeFunction;
-import com.github.sonus21.rqueue.web.view.DeadLetterQueuesFunction;
-import org.jtwig.environment.EnvironmentConfiguration;
-import org.jtwig.environment.EnvironmentConfigurationBuilder;
-import org.jtwig.spring.JtwigViewResolver;
-import org.jtwig.web.servlet.JtwigRenderer;
+import com.github.sonus21.rqueue.utils.pebble.ResourceLoader;
+import com.github.sonus21.rqueue.utils.pebble.RqueuePebbleExtension;
+import com.mitchellbosecke.pebble.PebbleEngine;
+import com.mitchellbosecke.pebble.spring.extension.SpringExtension;
+import com.mitchellbosecke.pebble.spring.servlet.PebbleViewResolver;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.beans.factory.config.ConfigurableBeanFactory;
@@ -172,20 +171,16 @@ public RqueueLockManager rqueueLockManager(RqueueStringDao rqueueStringDao) {
   }
 
   @Bean
-  public JtwigViewResolver rqueueViewResolver() {
-    EnvironmentConfiguration configuration =
-        EnvironmentConfigurationBuilder.configuration()
-            .functions()
-            .add(new DateTimeFunction())
-            .add(new DeadLetterQueuesFunction())
-            .and()
+  public PebbleViewResolver rqueueViewResolver() {
+    PebbleEngine pebbleEngine =
+        new PebbleEngine.Builder()
+            .extension(new RqueuePebbleExtension(), new SpringExtension(null))
+            .loader(new ResourceLoader())
             .build();
-    JtwigRenderer renderer = new JtwigRenderer(configuration);
-    JtwigViewResolver viewResolver = new JtwigViewResolver();
-    viewResolver.setRenderer(renderer);
-    viewResolver.setPrefix("classpath:/templates/rqueue/");
-    viewResolver.setSuffix(".html");
-    return viewResolver;
+    PebbleViewResolver resolver = new PebbleViewResolver(pebbleEngine);
+    resolver.setPrefix("templates/rqueue/");
+    resolver.setSuffix(".html");
+    return resolver;
   }
 
   @Bean

rqueue-core/src/main/java/com/github/sonus21/rqueue/core/EndpointRegistry.java

@@ -19,6 +19,7 @@
 import com.github.sonus21.rqueue.exception.OverrideException;
 import com.github.sonus21.rqueue.exception.QueueDoesNotExist;
 import com.github.sonus21.rqueue.listener.QueueDetail;
+import com.github.sonus21.rqueue.utils.PriorityUtils;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -44,6 +45,7 @@ private EndpointRegistry() {}
    * @param queueName queue name
    * @return queue detail
    * @throws QueueDoesNotExist this error is thrown when queue is not registered.
+   * @see #get(String, String)
    */
   public static QueueDetail get(String queueName) {
     QueueDetail queueDetail = queueNameToDetail.get(queueName);
@@ -53,6 +55,23 @@ public static QueueDetail get(String queueName) {
     return queueDetail;
   }
 
+  /**
+   * Get QueueDetail for the given queue, with priority
+   *
+   * @param queueName queue name
+   * @param priority  priority of this queue like critical, high
+   * @return queue detail
+   * @throws QueueDoesNotExist this error is thrown when queue is not registered.
+   */
+  public static QueueDetail get(String queueName, String priority) {
+    QueueDetail queueDetail =
+        queueNameToDetail.get(PriorityUtils.getQueueNameForPriority(queueName, priority));
+    if (queueDetail == null) {
+      throw new QueueDoesNotExist(queueName);
+    }
+    return queueDetail;
+  }
+
   public static void register(QueueDetail queueDetail) {
     synchronized (lock) {
       if (queueNameToDetail.containsKey(queueDetail.getName())) {

rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessagePoller.java

@@ -69,8 +69,7 @@ private void execute(QueueThread queueThread, QueueDetail queueDetail, RqueueMes
                 postProcessingHandler,
                 message,
                 queueDetail,
-                queueThread.getSemaphore()
-            ));
+                queueThread.getSemaphore()));
   }
 
   boolean shouldExit() {
@@ -96,7 +95,9 @@ void poll(int index, String queue, QueueDetail queueDetail, QueueThread queueThr
     }
     if (!acquired) {
       deactivate(index, queue, DeactivateType.SEMAPHORE_UNAVAILABLE);
-    } else if (isQueueActive(queue)) {
+      return;
+    }
+    if (isQueueActive(queue)) {
       try {
         RqueueMessage message = getMessage(queueDetail);
         log(Level.DEBUG, "Queue: {} Fetched Msg {}", null, queue, message);

rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/StrictPriorityPoller.java

@@ -59,6 +59,7 @@ class StrictPriorityPoller extends RqueueMessagePoller {
 
   private String getQueueToPoll() {
     long now = System.currentTimeMillis();
+    // starvation
     for (String queue : queues) {
       if (isQueueActive(queue)) {
         if (now - lastFetchedTime.get(queue) > Constants.MILLIS_IN_A_MINUTE) {

rqueue-core/src/main/java/com/github/sonus21/rqueue/metrics/RqueueQueueMetrics.java

@@ -79,4 +79,53 @@ public long getProcessingMessageCount(String queue) {
       return -1;
     }
   }
+
+  /**
+   * Get number of messages waiting for consumption
+   *
+   * @param queue    queue name
+   * @param priority priority of this queue
+   * @return -1 if queue is not registered otherwise message count
+   */
+  public long getPendingMessageCount(String queue, String priority) {
+    try {
+      QueueDetail queueDetail = EndpointRegistry.get(queue, priority);
+      return redisTemplate.getListSize(queueDetail.getQueueName());
+    } catch (QueueDoesNotExist e) {
+      return -1;
+    }
+  }
+
+  /**
+   * Get number of messages waiting in delayed queue, these messages would move to pending queue as
+   * soon as the scheduled time is reach.
+   *
+   * @param queue    queue name
+   * @param priority priority of this queue
+   * @return -1 if queue is not registered otherwise message count
+   */
+  public long getScheduledMessageCount(String queue, String priority) {
+    try {
+      QueueDetail queueDetail = EndpointRegistry.get(queue, priority);
+      return redisTemplate.getZsetSize(queueDetail.getDelayedQueueName());
+    } catch (QueueDoesNotExist e) {
+      return -1;
+    }
+  }
+
+  /**
+   * Get number of messages those are currently being processed
+   *
+   * @param queue    queue name
+   * @param priority priority of this queue
+   * @return -1 if queue is not registered otherwise message count
+   */
+  public long getProcessingMessageCount(String queue, String priority) {
+    try {
+      QueueDetail queueDetail = EndpointRegistry.get(queue, priority);
+      return redisTemplate.getZsetSize(queueDetail.getProcessingQueueName());
+    } catch (QueueDoesNotExist e) {
+      return -1;
+    }
+  }
 }

rqueue-core/src/main/java/com/github/sonus21/rqueue/web/view/DateTimeFunction.java rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/pebble/DateTimeFunction.java

@@ -14,22 +14,28 @@
  *
  */
 
-package com.github.sonus21.rqueue.web.view;
+package com.github.sonus21.rqueue.utils.pebble;
 
 import com.github.sonus21.rqueue.utils.DateTimeUtils;
-import org.jtwig.functions.FunctionRequest;
-import org.jtwig.functions.SimpleJtwigFunction;
+import com.mitchellbosecke.pebble.extension.Function;
+import com.mitchellbosecke.pebble.template.EvaluationContext;
+import com.mitchellbosecke.pebble.template.PebbleTemplate;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
 
-public class DateTimeFunction extends SimpleJtwigFunction {
+public class DateTimeFunction implements Function {
+  public static final String FUNCTION_NAME = "time";
 
   @Override
-  public String name() {
-    return "time";
+  public Object execute(
+      Map<String, Object> args, PebbleTemplate self, EvaluationContext context, int lineNumber) {
+    Long milli = (Long) args.get("milli");
+    return DateTimeUtils.formatMilliToString(milli);
   }
 
   @Override
-  public Object execute(FunctionRequest request) {
-    Long milli = (Long) request.getArguments().get(0);
-    return DateTimeUtils.formatMilliToString(milli);
+  public List<String> getArgumentNames() {
+    return Collections.singletonList("milli");
   }
 }