diff --git a/.github/workflows/java-ci.yaml b/.github/workflows/java-ci.yaml index 5722f37ec..7168ecd20 100644 --- a/.github/workflows/java-ci.yaml +++ b/.github/workflows/java-ci.yaml @@ -336,6 +336,85 @@ jobs: rqueue-core/build/reports/junit/xml if-no-files-found: ignore + nats_integration_test: + needs: build + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Java 21 + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "21" + cache: gradle + + - name: Expose Java 21 to Gradle toolchains + run: | + echo "JAVA_HOME=$JAVA_HOME" >> "$GITHUB_ENV" + echo "ORG_GRADLE_JAVA_INSTALLATIONS_PATHS=$JAVA_HOME" >> "$GITHUB_ENV" + java -version + javac -version + + - name: Set up Gradle + uses: gradle/actions/setup-gradle@v4 + + # Install + start nats-server directly (not via Docker — mirrors how the other CI + # jobs install redis-server). Tests detect NATS_RUNNING and connect to localhost + # instead of pulling a Testcontainers image. + - name: Install nats-server + run: | + NATS_VERSION=v2.10.22 + curl -sSL "https://github.com/nats-io/nats-server/releases/download/${NATS_VERSION}/nats-server-${NATS_VERSION}-linux-amd64.tar.gz" \ + | tar -xz -C /tmp + sudo mv "/tmp/nats-server-${NATS_VERSION}-linux-amd64/nats-server" /usr/local/bin/nats-server + nats-server --version + + - name: Start nats-server + run: | + mkdir -p /tmp/jetstream + nohup nats-server -js -sd /tmp/jetstream -p 4222 > /tmp/nats.log 2>&1 & + for i in $(seq 1 20); do + if (echo > /dev/tcp/127.0.0.1/4222) 2>/dev/null; then + echo "nats-server ready after ${i}s"; break + fi + sleep 1 + done + + - name: Run NATS tests + env: + NATS_RUNNING: "true" + NATS_URL: nats://127.0.0.1:4222 + run: ./gradlew :rqueue-nats:test :rqueue-spring-boot-starter:test :rqueue-spring:test -DincludeTags=nats + + - name: Upload nats-server log + if: always() + uses: actions/upload-artifact@v4 + with: + name: nats-server-log + path: /tmp/nats.log + if-no-files-found: ignore + + - name: Upload JaCoCo exec data + if: always() + uses: actions/upload-artifact@v4 + with: + name: coverage-nats + path: "**/build/reports/jacoco/*.exec" + if-no-files-found: error + + - name: Upload JUnit reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: nats-test-results + path: | + rqueue-nats/build/reports/junit/xml + rqueue-spring-boot-starter/build/reports/junit/xml + rqueue-spring/build/reports/junit/xml + if-no-files-found: ignore + coverage_report: needs: - unit_test @@ -343,6 +422,7 @@ jobs: - integration_test - redis_cluster_test - reactive_integration_test + - nats_integration_test runs-on: ubuntu-latest steps: - name: Checkout diff --git a/.gitignore b/.gitignore index 900e3694e..bbe70f853 100644 --- a/.gitignore +++ b/.gitignore @@ -32,4 +32,5 @@ hs_err_pid* /*/build/ /*/log -.DS_Store \ No newline at end of file +.DS_Store.claude/ +.claude/ diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..1e2588df7 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,17 @@ +# Project rules for AI tools + +## Commits + +**Never add `Co-Authored-By:` trailers crediting any AI tool to a git commit.** + +This includes Claude, Claude Code, GitHub Copilot, Cursor, Codeium, and any other AI assistant. Commits must list a human author only — AI tools are not co-authors. + +**You may add an `Assisted-By:` trailer naming the tool.** Example: + +``` +Assisted-By: Claude Code +``` + +Use a short, plain tool name. No emails, no URLs, no marketing taglines (`(1M context)`, `Generated with ...`). One trailer per commit is enough. + +If you are running as an agent or sub-agent and your default commit template includes a `Co-Authored-By:` for an AI tool, strip it before `git commit` and replace it with `Assisted-By: ` instead. This rule applies to all branches and to rewrites. diff --git a/README.md b/README.md index b5ae1f7e9..9aaab43d1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@
Rqueue Logo -

Rqueue: Redis-Backed Job Queue and Scheduler for Spring and Spring Boot

+

Rqueue: Job Queue and Scheduler for Spring and Spring Boot (Redis & NATS)

[![Coverage Status](https://coveralls.io/repos/github/sonus21/rqueue/badge.svg?branch=master)](https://coveralls.io/github/sonus21/rqueue?branch=master) @@ -8,10 +8,10 @@ [![Javadoc](https://javadoc.io/badge2/com.github.sonus21/rqueue-core/javadoc.svg)](https://javadoc.io/doc/com.github.sonus21/rqueue-core) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE) -**Rqueue** is a Redis-backed job queue and producer-consumer system for Spring and Spring Boot. It -supports both producers and consumers for background jobs, scheduled tasks, and event-driven -workflows, similar to Sidekiq or Celery, but fully integrated into the Spring programming model with -annotation-driven APIs and minimal setup. +**Rqueue** is a job queue and producer-consumer system for Spring and Spring Boot with pluggable +broker backends — **Redis** (default) and **NATS JetStream**. It supports producers and consumers +for background jobs, scheduled tasks, and event-driven workflows, similar to Sidekiq or Celery, +fully integrated into the Spring programming model with annotation-driven APIs and minimal setup.
@@ -29,7 +29,8 @@ annotation-driven APIs and minimal setup. * **Queues and routing** * Deduplicate messages using message IDs * Process priority workloads such as high, medium, and low - * Prioritize workloads with group-level queue priority and weighted, strict, or hard strict ordering + * Prioritize workloads with group-level queue priority and weighted, strict, or hard strict + ordering * Fan out the same message to multiple listeners * Poll messages in batches for higher throughput @@ -48,19 +49,21 @@ annotation-driven APIs and minimal setup. * Monitor in-flight, queued, and scheduled messages with metrics * Use the built-in web dashboard for queue visibility and latency insights -* **Redis and platform support** +* **Backend and platform support** + * Switch backends with a single property (`rqueue.backend=redis|nats`) * Use a separate Redis setup for Rqueue if needed * Support Redis standalone, Sentinel, and Cluster setups * Work with Lettuce for Redis Cluster * Support reactive Redis and Spring WebFlux + * Use NATS JetStream as a drop-in Redis replacement (add `rqueue-nats` and set `rqueue.backend=nats`) ### Requirements * Spring 5+, 6+, 7+ -* Java 1.8+,17, 21 -* Spring boot 2+,3+,4+ -* Lettuce client for Redis cluster -* Read master preference for Redis cluster +* Java 1.8+, 17, 21 +* Spring Boot 2+, 3+, 4+ +* **Redis backend (default):** Lettuce client; read-master preference for Redis Cluster +* **NATS backend:** NATS Server 2.2+ with JetStream enabled (`nats-server -js`); `rqueue-nats` on the classpath ## Getting Started @@ -94,6 +97,40 @@ from [Maven central](https://search.maven.org/search?q=g:com.github.sonus21%20AN No additional configurations are required, only dependency is required. +##### Spring Boot with NATS backend + +To use NATS JetStream instead of Redis, add `rqueue-nats` alongside the starter and set +`rqueue.backend=nats` in `application.properties`: + +* Gradle + ```groovy + implementation 'com.github.sonus21:rqueue-spring-boot-starter:4.0.0-RELEASE' + implementation 'com.github.sonus21:rqueue-nats:4.0.0-RELEASE' + ``` +* Maven + ```xml + + com.github.sonus21 + rqueue-spring-boot-starter + 4.0.0-RELEASE + + + com.github.sonus21 + rqueue-nats + 4.0.0-RELEASE + + ``` + +Then in `application.properties`: +```properties +rqueue.backend=nats +rqueue.nats.connection.url=nats://localhost:4222 +``` + +No `RedisConnectionFactory` bean is required. Start a JetStream-enabled NATS server with +`nats-server -js` and the application is ready. See the [NATS backend](#nats-backend) section +below for streams, KV buckets, and advanced configuration. + --- #### Spring Framework @@ -236,7 +273,7 @@ public class MessageListener { // checkin job example @RqueueListener(value = "chat-indexing-weekly", priority = "5", priorityGroup = "chat") public void onMessage(ChatIndexing chatIndexing, - @Header(RqueueMessageHeaders.JOB) com.github.sonus21.rqueue.core.Job job) { + @Header(RqueueMessageHeaders.JOB) com.github.sonus21.rqueue.core.Job job) { log.info("ChatIndexing message: {}", chatIndexing); job.checkIn("Chat indexing..."); } @@ -267,6 +304,185 @@ Micrometer based dashboard for queue --- +## NATS backend + +Rqueue can use NATS JetStream as the message broker instead of Redis by setting +`rqueue.backend=nats` and including the `rqueue-nats` module on the classpath. State that Redis +stores in keys, hashes, and sorted-sets is mapped onto JetStream **streams** (for messages) and +JetStream **KV buckets** (for everything else). Both are provisioned **once at startup** — +streams by `NatsStreamValidator` on `RqueueBootstrapEvent`, KV buckets by `NatsKvBucketValidator` +on the `Connection` bean — so the publish / pop hot path never pays a `getStreamInfo` round-trip +to confirm the stream exists. As long as the JetStream credentials allow `add_stream` / +`kv_create`, nothing needs to be created ahead of time. For locked-down accounts see the +"Pre-creating streams" / "Pre-creating buckets" subsections below. + +### Streams per queue + +Each registered queue produces **one main stream**, **one DLQ stream** (when +`rqueue.nats.autoCreateDlqStream=true`, the default), and **one extra stream per priority +sub-queue** the queue declares. Only the main queue has a DLQ — priority sub-queues fan out to +their own streams but share the parent queue's DLQ wiring through `RqueueExecutor`. + +| Queue shape | Stream count | Names (with default prefixes) | +|---------------------------------|--------------|---------------------------------------------------------------------------------------------------| +| Plain queue, DLQ on (default) | 2 | `rqueue-js-`, `rqueue-js--dlq` | +| Plain queue, DLQ off | 1 | `rqueue-js-` | +| Queue with N priorities, DLQ on | N + 2 | `rqueue-js-`, `rqueue-js--` … `rqueue-js--`, `rqueue-js--dlq` | + +The naming scheme is `[-][]`, configurable via +`rqueue.nats.naming.streamPrefix` (default `rqueue-js-`) and `rqueue.nats.naming.dlqSuffix` +(default `-dlq`). The `-js-` segment makes Rqueue's message streams easy to distinguish at a +glance from the JetStream-backed KV buckets below (which keep the plain `rqueue-` prefix because +that's the operator-facing bucket name, not a stream name) and from anything else sharing the +JetStream account. Subjects follow the same shape with `.` separators: +`[.][]` (default subject prefix +`rqueue.js.`). Stream defaults (replicas, storage, retention, duplicate window, max msgs/bytes) +come from `rqueue.nats.stream.*`. + +#### Pre-creating streams (restricted JetStream accounts) + +For deployments where the application credentials cannot run `add_stream` at runtime, set +`rqueue.nats.autoCreateStreams=false` and pre-create every stream the application needs. +`NatsStreamValidator` walks `EndpointRegistry` on `RqueueBootstrapEvent` and verifies that +every main stream, every priority sub-queue stream, and every DLQ stream (for queues whose +listener declared a DLQ) exists. If any are missing it aborts boot with one +`IllegalStateException` listing all of them — operator-actionable failure at startup, not a +"stream not found" on first enqueue. + +The streams to pre-create follow the table above. For a queue `orders` with priorities +`high` / `low` and a DLQ: + +```sh +nats stream add rqueue-js-orders --subjects rqueue.js.orders ... +nats stream add rqueue-js-orders-high --subjects rqueue.js.orders.high ... +nats stream add rqueue-js-orders-low --subjects rqueue.js.orders.low ... +nats stream add rqueue-js-orders-dlq --subjects rqueue.js.orders.dlq ... +``` + +Consumers (durable pull consumers) are still created lazily — the broker calls +`ensureConsumer` once per `(stream, consumerName)` pair on the cold path of the first pop and +caches the bind in-process, so there's no per-pop RTT after warm-up. Set +`rqueue.nats.autoCreateConsumers=false` to fail-fast on missing consumers instead of creating +them. + +### KV buckets (one set, shared across all queues) + +State that Redis stores in keys, hashes, and sorted-sets is mapped onto JetStream **KV buckets** — +one bucket per concern, **not per queue** (per-queue scoping is done via key prefix). All buckets +use the default replicas / storage settings of the JetStream account unless noted; per-entry TTL +relies on the bucket's `ttl` (NATS' name for `maxAge`), which is set once at bucket creation. + +| Bucket name | Purpose | TTL behaviour | Created in | +|----------------------------|-------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `rqueue-queue-config` | Per-queue `QueueConfig` records (registered queues, DLQ wiring, flags). | No TTL. Entries persist until explicitly overwritten. | [`NatsRqueueSystemConfigDao`](rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/dao/NatsRqueueSystemConfigDao.java) (`@Conditional(NatsBackendCondition)`) | +| `rqueue-jobs` | `RqueueJob` execution history per message id. | TTL captured from the first `createJob`/`save` call's `expiry` argument; bucket-level so it applies uniformly. | [`NatsRqueueJobDao`](rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/dao/NatsRqueueJobDao.java) | +| `rqueue-locks` | Distributed locks (scheduler leadership, message-level locks). | TTL captured from the first `acquireLock` call's `duration` argument. | [`NatsRqueueLockManager`](rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/lock/NatsRqueueLockManager.java) | +| `rqueue-message-metadata` | Per-message metadata (delivery status, retry count, dead-letter flags). | No TTL at the bucket. Per-write `ttl` arguments are ignored on this v1 impl. | [`NatsRqueueMessageMetadataService`](rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/service/NatsRqueueMessageMetadataService.java) | +| `rqueue-workers` | Worker process info (host, pid, version, last-seen). | TTL = `rqueue.workerRegistry.workerTtl` (captured on first heartbeat). | [`NatsWorkerRegistryStore`](rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/worker/NatsWorkerRegistryStore.java) | +| `rqueue-worker-heartbeats` | Per-(queue, worker) heartbeats. Keys flattened as `__`. | TTL = `rqueue.workerRegistry.queueTtl` (captured on first refresh; falls back to 1 h if registry not enabled). | [`NatsWorkerRegistryStore`](rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/worker/NatsWorkerRegistryStore.java) | + +### How buckets are configured + +- **Lazy, code-driven creation.** Each store / dao calls `kvm.create(KeyValueConfiguration...)` + the first time it is touched after startup. There is no `application.yml` switch to disable + this, and there is no provisioning step you need to run by hand — but the JetStream account + used by your `Connection` bean must have permission to create KV buckets (i.e. JetStream must + be enabled and account limits must allow it). +- **TTL is fixed at bucket creation.** All buckets that take a `ttl` snapshot the value at + creation. Changing the corresponding rqueue property after the bucket exists has no effect + until the bucket is deleted out-of-band and recreated. This matches NATS KV semantics — the + bucket's `maxAge` is immutable. +- **No bucket per queue.** All queues share the same buckets above; per-queue scoping is done + via the key prefix (`rqueue.workerRegistry.queueKey(queueName)`, etc.). +- **Connection wiring.** The `io.nats.client.Connection` bean comes from + [ + `RqueueNatsAutoConfig`](rqueue-spring-boot-starter/src/main/java/com/github/sonus21/rqueue/spring/boot/RqueueNatsAutoConfig.java) + (Spring Boot) when `rqueue.backend=nats` and `io.nats.client.JetStream` is on the classpath. + All KV stores receive that same `Connection` and call `connection.keyValueManagement()` / + `connection.keyValue(name)` against it. + +### Pre-creating buckets (restricted JetStream accounts) + +In managed or locked-down JetStream deployments the credentials your application uses may not +have permission to create KV buckets at runtime. In that case the lazy `kvm.create(...)` call +on first use will fail with `JetStreamApiException` ("permission violation" or "stream not +found"), and depending on the call site the failure may be logged and swallowed (registry, +metadata) or surface as a missing record. + +For these deployments, **set `rqueue.nats.autoCreateKvBuckets=false`** and pre-create the +buckets manually. With the flag off, Rqueue's `NatsKvBucketValidator` walks every bucket in +`NatsKvBuckets.ALL_BUCKETS` via `kvm.getStatus(name)` and aborts boot with an +`IllegalStateException` listing every missing bucket — converting a late-binding "permission +violation on first use" failure into a deterministic startup failure with operator-facing +remediation. Two independent mechanisms guarantee it runs before any KV-touching bean: + +1. **Inline call in `natsConnection`** (Spring Boot path). The auto-config invokes + `NatsKvBucketValidator.validate(connection, ...)` inside the `Connection` bean factory + method, so the bean cannot be returned — and no dependent bean instantiated — until + validation has succeeded. +2. **`@DependsOn("natsKvBucketValidator")`** on every NATS-backed bean + (`NatsRqueueSystemConfigDao`, `NatsRqueueJobDao`, `NatsRqueueLockManager`, + `NatsRqueueMessageMetadataService`, `NatsWorkerRegistryStore`, plus the `@Bean` factory + for `WorkerRegistryStore`). Spring resolves `@DependsOn` before constructor injection, so + the validator's `InitializingBean#afterPropertiesSet` fires before any KV bean is built. + The validator bean itself is declared in `RqueueNatsAutoConfig` and reads the flag from + `RqueueNatsProperties` — `rqueue-nats` never reads `rqueue.nats.*` keys directly. Plain + (non-Boot) Spring users who skip the auto-config can declare an equivalent bean themselves + passing `new NatsKvBucketValidator(connection, autoCreate)`. + +Spring's `@Order`/`@Priority` only affect collection injection ordering, not bean creation +order, so anchoring on the dependency root (`Connection`) and on `@DependsOn` is what +guarantees the right run order. + +```yaml +rqueue: + backend: nats + nats: + autoCreateKvBuckets: false # validate only; never call kvm.create() at runtime +``` + +The commands below assume the [`nats` CLI](https://docs.nats.io/using-nats/nats-tools/nats_cli) +is configured against the same account and creds your application uses. Substitute your own +values for replicas, storage, and TTL; the values shown match the defaults Rqueue would use if +it created the bucket itself. + +```bash +# State that must persist (no TTL). +nats kv add rqueue-queue-config --replicas=3 --storage=file +nats kv add rqueue-message-metadata --replicas=3 --storage=file + +# Job history. Use the same value as rqueue.job.durability (default 7 days). +nats kv add rqueue-jobs --replicas=3 --storage=file --ttl=7d + +# Distributed locks. Use a value at least as large as your longest expected lock hold. +nats kv add rqueue-locks --replicas=3 --storage=file --ttl=10m + +# Worker registry. Match rqueue.workerRegistry.workerTtl / queueTtl exactly. +nats kv add rqueue-workers --replicas=3 --storage=file --ttl=5m +nats kv add rqueue-worker-heartbeats --replicas=3 --storage=file --ttl=10m +``` + +Once the buckets exist, Rqueue's lazy initialiser short-circuits — `kvm.getStatus(name)` returns +non-null and the existing bucket is opened, no `create` call is made. The application +credentials only need read/write on the buckets, not management privileges. + +### Re-creating a bucket with new settings + +If you need to change a bucket's TTL or replication settings after deployment, delete the +bucket via the NATS CLI and either let Rqueue recreate it on the next startup (open accounts) +or recreate it yourself with the new flags (restricted accounts): + +```bash +nats kv del rqueue-worker-heartbeats --force +nats kv add rqueue-worker-heartbeats --replicas=3 --storage=file --ttl=20m +``` + +Be aware that any data in the bucket is lost (which is acceptable for the worker registry and +locks, but **not** for `rqueue-queue-config` — back it up first if you have configured queues +through the dashboard). + +--- + ## Status Rqueue is stable and production ready, processing millions of messages daily in production @@ -328,7 +544,8 @@ signing.secretKeyRingFile=/Users/sonu/.gnupg/secring.gpg generate this as `gpg - You are most welcome for any pull requests for any feature/bug/enhancement. You would need Java8 and gradle to start with. In root `build.gradle` file comment out spring related versions, or set -environment variables for Spring versions. You can use [module, class and other diagrams](https://sourcespy.com/github/sonus21rqueue/) +environment variables for Spring versions. You can +use [module, class and other diagrams](https://sourcespy.com/github/sonus21rqueue/) to familiarise yourself with the project. **Please format your code with Palantir Java Format using `./gradlew formatJava`.** @@ -337,7 +554,8 @@ to familiarise yourself with the project. * Documentation: [https://sonus21.github.io/rqueue](https://sonus21.github.io/rqueue) * Releases: [https://github.com/sonus21/rqueue/releases](https://github.com/sonus21/rqueue/releases) -* Issue tracker: [https://github.com/sonus21/rqueue/issues](https://github.com/sonus21/rqueue/issues) +* Issue + tracker: [https://github.com/sonus21/rqueue/issues](https://github.com/sonus21/rqueue/issues) * Maven Central: * [https://repo1.maven.org/maven2/com/github/sonus21/rqueue-spring](https://repo1.maven.org/maven2/com/github/sonus21/rqueue-spring) * [https://repo1.maven.org/maven2/com/github/sonus21/rqueue-spring-boot-starter](https://repo1.maven.org/maven2/com/github/sonus21/rqueue-spring-boot-starter) diff --git a/build.gradle b/build.gradle index 7dcae112f..c8c9ffa76 100644 --- a/build.gradle +++ b/build.gradle @@ -65,6 +65,8 @@ ext { // database lettuceVersion = "7.2.1.RELEASE" + natsVersion = "2.25.2" + testcontainersVersion = "1.20.4" jakartaAnnotationVersion = "3.0.0" jakartaPersistenceVersion = "3.2.0" hibernateCoreVersion = "7.2.1.Final" @@ -83,7 +85,7 @@ ext { subprojects { group = "com.github.sonus21" - version = "4.0.0-RC3" + version = "4.0.0-RC4" dependencies { // https://mvnrepository.com/artifact/org.springframework/spring-messaging diff --git a/docs/_config.yml b/docs/_config.yml index 914a8df88..6c22d4173 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -14,7 +14,7 @@ # You can create any custom variable you would like, and they will be accessible # in the templates via {{ site.myvariable }}. title: Rqueue -description: Library for Async Processing using Redis +description: Library for Async Processing using Redis or NATS JetStream baseurl: "/rqueue" # the subpath of your site, e.g. /blog url: "https://sonus21.github.io/rqueue" # the base hostname & protocol for your site, e.g. http://example.com diff --git a/docs/configuration/nats-configuration.md b/docs/configuration/nats-configuration.md new file mode 100644 index 000000000..ef1eb07b8 --- /dev/null +++ b/docs/configuration/nats-configuration.md @@ -0,0 +1,379 @@ +--- +layout: default +title: NATS Configuration +parent: Configuration +nav_order: 3 +--- + +# NATS Configuration +{: .no_toc } + +
+ Table of contents + {: .text-delta } +1. TOC +{:toc} +
+ +--- + +Rqueue supports **NATS JetStream** as a drop-in replacement for Redis. All listener, +producer, and middleware APIs work identically — the only changes required are the +dependency and two properties. + +{: .warning } +The NATS backend does not support delayed enqueue, scheduled messages, or periodic/cron +jobs. Calls to `enqueueIn`, `enqueueAt`, and `enqueuePeriodic` throw +`UnsupportedOperationException` at runtime. Use the Redis backend for workloads that +need scheduling. + +--- + +## Quick Setup + +### 1. Add the dependency + +Add `rqueue-nats` alongside `rqueue-spring-boot-starter`: + +**Gradle** +```groovy +implementation 'com.github.sonus21:rqueue-spring-boot-starter:4.0.0-RELEASE' +implementation 'com.github.sonus21:rqueue-nats:4.0.0-RELEASE' +``` + +**Maven** +```xml + + com.github.sonus21 + rqueue-spring-boot-starter + 4.0.0-RELEASE + + + com.github.sonus21 + rqueue-nats + 4.0.0-RELEASE + +``` + +### 2. Configure `application.properties` + +```properties +rqueue.backend=nats +rqueue.nats.connection.url=nats://localhost:4222 +``` + +No `RedisConnectionFactory` bean is needed. All Rqueue listener, producer, and +middleware annotations work without any code changes. + +### 3. Start NATS with JetStream enabled + +```sh +# native binary +nats-server -js + +# Docker +docker run -p 4222:4222 nats:latest -js +``` + +At startup, Rqueue's `NatsStreamValidator` and `NatsKvBucketValidator` provision all +required streams and KV buckets automatically. + +--- + +## Connection Properties + +All connection properties are under the `rqueue.nats.connection` prefix. + +| Property | Type | Default | Description | +|---|---|---|---| +| `url` | `String` | `nats://localhost:4222` | Single NATS server URL. | +| `username` | `String` | — | Username for username/password authentication. | +| `password` | `String` | — | Password for username/password authentication. | +| `token` | `String` | — | Token for token-based authentication. | +| `credentials-path` | `String` | — | Path to a `.creds` file for NKey/JWT authentication. | +| `tls` | `boolean` | `false` | Enable TLS for the connection. | +| `connection-name` | `String` | — | Logical name visible in NATS server monitoring. | +| `connect-timeout` | `Duration` | (client default) | Maximum time to wait for initial connection. | +| `reconnect-wait` | `Duration` | (client default) | Time to wait between reconnect attempts. | +| `max-reconnects` | `int` | `-1` (unlimited) | Maximum reconnect attempts. | +| `ping-interval` | `Duration` | (client default) | Interval between server pings. | + +### Authentication examples + +**Token authentication** +```properties +rqueue.nats.connection.token=s3cr3t +``` + +**Username / password** +```properties +rqueue.nats.connection.username=rqueue +rqueue.nats.connection.password=s3cr3t +``` + +**NKey / JWT credentials file** +```properties +rqueue.nats.connection.credentials-path=/etc/nats/rqueue.creds +``` + +### Connection resilience + +```properties +# Retry for up to 10 minutes (120 attempts × 5 s wait) +rqueue.nats.connection.max-reconnects=120 +rqueue.nats.connection.reconnect-wait=5s +``` + +Set `max-reconnects=-1` (the default) for unlimited retries in production — NATS +reconnects silently to any cluster node without dropping in-flight messages. + +--- + +## Stream Properties + +Each registered queue maps to one or more JetStream streams. The defaults below apply +to every stream Rqueue creates. All properties are under `rqueue.nats.stream`. + +| Property | Type | Default | Description | +|---|---|---|---| +| `replicas` | `int` | `1` | Number of stream replicas. Must not exceed the number of JetStream-enabled servers in the cluster. | +| `storage` | `String` | `FILE` | Storage backend: `FILE` (durable) or `MEMORY` (faster, lost on restart). | +| `retention` | `String` | `LIMITS` | Retention policy: `LIMITS`, `INTEREST`, or `WORK_QUEUE`. | +| `max-age` | `Duration` | `14d` | Maximum age of messages before automatic removal. | +| `max-bytes` | `long` | `-1` (unlimited) | Maximum total stream size in bytes. | +| `max-messages` | `long` | `-1` (unlimited) | Maximum number of messages in the stream. | +| `discard-policy` | `String` | `OLD` | What to discard when limits are hit: `OLD` (oldest messages) or `NEW` (reject new publishes). | +| `duplicate-window` | `Duration` | `2m` | Server-side dedup window for the `Nats-Msg-Id` header. | + +{: .note } +`duplicate-window` must be less than or equal to `max-age`. Set it to cover the +maximum time a publisher might retry the same message ID (e.g. after a crash recovery). + +### Retention policy guide + +| Value | When to use | +|---|---| +| `LIMITS` (default) | General-purpose queues. Messages are kept until age/size limits are hit. | +| `INTEREST` | Fan-out / pub-sub patterns. Messages are removed once every active consumer has acked. | +| `WORK_QUEUE` | Lowest storage overhead. Message is removed as soon as any consumer acks it. Use for non-fan-out queues where exactly-once delivery per message is the goal. | + +### Three-replica production setup + +```properties +rqueue.nats.stream.replicas=3 +rqueue.nats.stream.storage=FILE +rqueue.nats.stream.max-age=7d +rqueue.nats.stream.duplicate-window=5m +``` + +--- + +## Consumer Properties + +Consumer properties control the durable pull consumers Rqueue creates for each +`(queue, consumerName)` pair. All properties are under `rqueue.nats.consumer`. + +| Property | Type | Default | Description | +|---|---|---|---| +| `ack-wait` | `Duration` | `30s` | Time the server waits for an ack before redelivering. Must be longer than your slowest message handler. | +| `max-deliver` | `long` | `3` | Delivery attempts before a message is forwarded to the DLQ. | +| `max-ack-pending` | `long` | `1000` | Maximum unacked messages a consumer can hold before the server stops delivering. | +| `fetch-wait` | `Duration` | `2s` | How long `pop()` blocks waiting for messages before returning empty. | + +{: .note } +`ack-wait` is the most important consumer setting. If a message handler takes longer +than `ack-wait`, the server redelivers the message to another consumer, causing +duplicate processing. Set it to at least 2× your 99th-percentile handler latency. + +### Tuning for slow handlers + +```properties +# Handlers can take up to 5 minutes +rqueue.nats.consumer.ack-wait=6m +# Give each message 5 attempts before DLQ +rqueue.nats.consumer.max-deliver=5 +``` + +### Tuning for high-throughput queues + +```properties +# Allow more unacked messages in-flight +rqueue.nats.consumer.max-ack-pending=5000 +# Reduce idle wait to pick up bursts faster +rqueue.nats.consumer.fetch-wait=500ms +``` + +--- + +## Naming Properties + +Naming properties control how stream and subject names are derived from queue names. +All properties are under `rqueue.nats.naming`. + +| Property | Type | Default | Description | +|---|---|---|---| +| `stream-prefix` | `String` | `rqueue-js-` | Prefix for every JetStream stream name. | +| `subject-prefix` | `String` | `rqueue.js.` | Prefix for every JetStream subject. | +| `dlq-suffix` | `String` | `-dlq` | Suffix appended to stream and subject names for DLQ streams. | + +For a queue named `orders` with priority sub-queues `high` and `low` and a DLQ, the +default naming produces: + +| Purpose | Stream name | Subject | +|---|---|---| +| Main queue | `rqueue-js-orders` | `rqueue.js.orders` | +| Priority: high | `rqueue-js-orders-high` | `rqueue.js.orders.high` | +| Priority: low | `rqueue-js-orders-low` | `rqueue.js.orders.low` | +| Dead-letter queue | `rqueue-js-orders-dlq` | `rqueue.js.orders.dlq` | + +{: .note } +Change the prefixes before the first deployment. Renaming them afterward requires +manually migrating or recreating all streams. + +--- + +## Auto-Provisioning + +### Streams (`rqueue.nats.auto-create-streams`) + +When `true` (default), `NatsStreamValidator` creates every required stream at startup, +immediately after all `@RqueueListener` methods are registered and before message +pollers start. This means the hot publish/pop path never pays a `getStreamInfo` +round-trip to confirm stream existence. + +Set to `false` for accounts where credentials lack `add_stream` permission. The +validator will instead check that every required stream exists and abort boot with a +clear `IllegalStateException` listing all missing streams — a deterministic startup +failure rather than a `stream not found` error on first enqueue. + +### DLQ streams (`rqueue.nats.auto-create-dlq-stream`) + +When `true`, a dead-letter stream is automatically created for every queue whose +`@RqueueListener` declares a `deadLetterQueue`. Default is `false` — enable it when +you want the DLQ stream provisioned alongside the main stream without pre-creating it +manually. + +### Consumers (`rqueue.nats.auto-create-consumers`) + +When `true` (default), durable pull consumers are created lazily on the first `pop` +call for each `(stream, consumerName)` pair and the subscription is cached in-process. +There is no per-pop round-trip after warm-up. + +Set to `false` to fail-fast on missing consumers instead of creating them. + +### KV buckets (`rqueue.nats.auto-create-kv-buckets`) + +Rqueue uses six shared KV buckets for state that Redis stores in keys, hashes, and +sorted sets: + +| Bucket | Purpose | TTL | +|---|---|---| +| `rqueue-queue-config` | Per-queue configuration and DLQ wiring | None (persists) | +| `rqueue-jobs` | Job execution history per message ID | `rqueue.message.durability` (default 7 days) | +| `rqueue-locks` | Distributed locks for scheduler leadership | Set per lock acquisition | +| `rqueue-message-metadata` | Per-message delivery status and retry count | None | +| `rqueue-workers` | Worker process info (host, PID, last-seen) | `rqueue.worker.registry.worker.ttl` (default 300 s) | +| `rqueue-worker-heartbeats` | Per-(queue, worker) heartbeats | `rqueue.worker.registry.queue.ttl` (default 3600 s) | + +When `auto-create-kv-buckets=true` (default), each store lazily creates its bucket on +first use. When set to `false`, `NatsKvBucketValidator` walks every bucket and aborts +boot listing any that are missing. + +--- + +## Locked-Down JetStream Accounts + +For deployments where application credentials cannot call `add_stream` or `kv_create` +at runtime, disable all auto-provisioning and pre-create every resource before +starting the application: + +```properties +rqueue.nats.auto-create-streams=false +rqueue.nats.auto-create-consumers=false +rqueue.nats.auto-create-dlq-stream=false +rqueue.nats.auto-create-kv-buckets=false +``` + +### Pre-creating streams + +For a queue `orders` with priorities `high` / `low` and a DLQ: + +```sh +nats stream add rqueue-js-orders \ + --subjects "rqueue.js.orders" \ + --storage file --replicas 3 --retention limits + +nats stream add rqueue-js-orders-high \ + --subjects "rqueue.js.orders.high" \ + --storage file --replicas 3 --retention limits + +nats stream add rqueue-js-orders-low \ + --subjects "rqueue.js.orders.low" \ + --storage file --replicas 3 --retention limits + +nats stream add rqueue-js-orders-dlq \ + --subjects "rqueue.js.orders.dlq" \ + --storage file --replicas 3 --retention limits +``` + +### Pre-creating KV buckets + +Match TTL values to your `rqueue.worker.registry.*` settings: + +```sh +# Persistent state — no TTL +nats kv add rqueue-queue-config --replicas=3 --storage=file +nats kv add rqueue-message-metadata --replicas=3 --storage=file + +# Job history — match rqueue.message.durability (default 7 days) +nats kv add rqueue-jobs --replicas=3 --storage=file --ttl=7d + +# Locks — cover your longest expected lock hold +nats kv add rqueue-locks --replicas=3 --storage=file --ttl=10m + +# Worker registry — match rqueue.worker.registry.worker.ttl (default 300 s) +nats kv add rqueue-workers --replicas=3 --storage=file --ttl=5m + +# Queue heartbeats — match rqueue.worker.registry.queue.ttl (default 3600 s) +nats kv add rqueue-worker-heartbeats --replicas=3 --storage=file --ttl=1h +``` + +{: .warning } +KV bucket TTLs are immutable after creation. To change a TTL, delete the bucket and +recreate it. Do not delete `rqueue-queue-config` without backing it up first — it +stores all registered queue configurations. + +--- + +## Inspecting Runtime State + +Use the `nats` CLI to inspect what Rqueue has created: + +```sh +# List all Rqueue streams +nats stream ls | grep rqueue-js- + +# Show message counts per queue +nats stream info rqueue-js-orders + +# List KV buckets +nats kv ls | grep rqueue- + +# Inspect queue configuration +nats kv get rqueue-queue-config orders +``` + +--- + +## Limitations + +| Feature | Redis backend | NATS backend | +|---|---|---| +| `enqueueIn` (delayed) | Supported | Not supported (throws `UnsupportedOperationException`) | +| `enqueueAt` (scheduled) | Supported | Not supported | +| `enqueuePeriodic` (cron) | Supported | Not supported | +| `priorityGroup` weighting | Full support | Boot warning; weighting not honored | +| Elastic `concurrency` (min < max) | Supported | Falls back to `max` | +| `@RqueueHandler(primary)` | Supported | Ignored with boot warning | +| Dashboard charts and message browse | Full support | Queue sizes only; charts and message browse unavailable | +| Reactive listener container | Supported | Enqueue side only | diff --git a/docs/index.md b/docs/index.md index 0be065738..854f4e709 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2,18 +2,19 @@ layout: default title: Home nav_order: 1 -description: Rqueue Redis Based Async Message Processor +description: Rqueue Job Queue and Scheduler for Spring — Redis and NATS JetStream backends permalink: / --- -# Rqueue | Redis-Backed Job Queue and Scheduler For Spring Framework +# Rqueue | Job Queue and Scheduler For Spring Framework (Redis & NATS) {: .fs-4 } -Rqueue is a Redis-backed job queue and producer-consumer system for Spring and Spring Boot. It -supports both producers and consumers for background jobs, scheduled tasks, and event-driven -workflows, similar to Sidekiq or Celery, while staying fully integrated with the Spring programming -model through annotation-driven APIs and minimal setup. +Rqueue is a job queue and producer-consumer system for Spring and Spring Boot with pluggable +broker backends — **Redis** (default) and **NATS JetStream**. It supports producers and consumers +for background jobs, scheduled tasks, and event-driven workflows, similar to Sidekiq or Celery, +while staying fully integrated with the Spring programming model through annotation-driven APIs +and minimal setup. {: .fs-6 .fw-300 } @@ -55,10 +56,12 @@ model through annotation-driven APIs and minimal setup. * Use the built-in web dashboard for queue visibility, worker activity, and message operations * Override message ID generation with a custom `RqueueMessageIdGenerator` bean -* **Redis and platform support** +* **Backend and platform support** + * Switch backends with a single property (`rqueue.backend=redis|nats`) * Support Redis standalone, Sentinel, and Cluster setups * Support reactive Redis and Spring WebFlux * Keep Redis configuration flexible for different deployment models + * Use NATS JetStream as a drop-in Redis replacement (add `rqueue-nats` and set `rqueue.backend=nats`) ### Requirements @@ -66,8 +69,8 @@ model through annotation-driven APIs and minimal setup. * Spring Boot 3+, 4+ * Java 21+ * Spring Reactive -* Lettuce client for Redis cluster -* Read master preference for Redis cluster +* **Redis backend (default):** Lettuce client; read-master preference for Redis Cluster +* **NATS backend:** NATS Server 2.2+ with JetStream enabled (`nats-server -js`); `rqueue-nats` on the classpath ## Getting Started @@ -80,7 +83,8 @@ inconsistencies. Queues should **only** be created when using Rqueue as a produc {: .highlight } The Rqueue GitHub repository includes several sample applications for local testing and demonstration: -* [Rqueue Spring Boot Example](https://github.com/sonus21/rqueue/tree/master/rqueue-spring-boot-example) +* [Rqueue Spring Boot Example](https://github.com/sonus21/rqueue/tree/master/rqueue-spring-boot-example) — Redis backend +* [Rqueue Spring Boot NATS Example](https://github.com/sonus21/rqueue/tree/master/rqueue-spring-boot-nats-example) — NATS JetStream backend * [Rqueue Spring Boot Reactive Example](https://github.com/sonus21/rqueue/tree/master/rqueue-spring-boot-reactive-example) * [Rqueue Spring Example](https://github.com/sonus21/rqueue/tree/master/rqueue-spring-example) @@ -161,6 +165,25 @@ public class Application { --- +### NATS JetStream Backend + +To use NATS JetStream instead of Redis, add `rqueue-nats` alongside the starter, set +`rqueue.backend=nats`, and point `rqueue.nats.connection.url` at a JetStream-enabled server. +No `RedisConnectionFactory` bean is required. All listener, producer, and middleware APIs work +without any code changes. + +```properties +rqueue.backend=nats +rqueue.nats.connection.url=nats://localhost:4222 +``` + +{: .note } +See [NATS Configuration](configuration/nats-configuration) for the full reference — connection +options, stream defaults, consumer tuning, naming, auto-provisioning, locked-down account setup, +and feature limitations. + +--- + {: .highlight } Once Rqueue is configured, you can use its methods and annotations consistently diff --git a/docs/static/docs-overrides.css b/docs/static/docs-overrides.css new file mode 100644 index 000000000..10b47a6fa --- /dev/null +++ b/docs/static/docs-overrides.css @@ -0,0 +1,12 @@ +/* Dark-mode note callouts read better with a warm accent than the default blue/purple tone. */ +.note { + background: linear-gradient(135deg, rgba(84, 63, 26, 0.82), rgba(48, 39, 22, 0.94)); + border-left-color: #d7a54a; + box-shadow: inset 0 1px 0 rgba(255, 224, 158, 0.06); +} + +.note .label, +.note-title, +.note > :first-child { + color: #f0bf68; +} diff --git a/nats-task-v2.md b/nats-task-v2.md new file mode 100644 index 000000000..ec224e8bd --- /dev/null +++ b/nats-task-v2.md @@ -0,0 +1,72 @@ +# NATS backend — v2 task tracker + +## v1 status: COMPLETE + +All v1 items are done and 360 unit tests pass. Branch `nats-backend` is ready to merge. + +--- + +## v2 pending items + +### 1. Web dashboard — NATS gaps + +Controllers are no longer Redis-gated but several operations throw `BackendCapabilityException` (HTTP 501) on NATS. The front-end should hide unsupported panels proactively instead of relying on 501s. + +- Expose `GET /rqueue/api/capabilities` returning the `Capabilities` record so the UI can conditionally hide panels. +- Extend `Capabilities` with dashboard-op flags: `supportsCharts`, `supportsMessageBrowse`, `supportsAdminMove`. +- Wire the flags into Pebble templates (scheduled panel, cron jobs panel, chart panel already have `hideScheduledPanel` / `hideCronJobs` hooks in `DataViewResponse`). + +Affected services that throw on NATS today: +- `RqueueDashboardChartServiceImpl` — time-series charts (no equivalent in JetStream) +- `RqueueUtilityServiceImpl` — move/enqueue admin ops +- `NatsMessageBrowsingRepository.viewData` — positional message browse + +### 2. Reactive listener container + +Only the enqueue side is reactive in v1. The listener/pop side still uses blocking `BrokerMessagePoller` threads. + +- Implement a `ReactiveMessagePoller` using `js.publishAsync` + Project Reactor for the NATS path. +- Gate behind `@Conditional(ReactiveEnabled.class)` + `NatsBackendCondition`. + +### 3. Delayed / scheduled / cron messages on NATS + +`enqueueWithDelay` throws `UnsupportedOperationException` in v1. Options: + +- Use NATS JetStream `MaxAge` + a separate "delay bucket" stream per delay tier (coarse buckets: 1s, 5s, 30s, 5m, 1h). +- Or implement a lightweight delay-scheduler sidecar using KV TTL expiry events. + +### 4. `priorityGroup` weighting on NATS + +In v1, cross-queue `priorityGroup` weighting logs a boot WARN and is not honored. `BrokerMessagePoller` spawns one thread per `(queue, consumerName, priority)` triple at fixed weight. + +- Implement weighted round-robin across pollers sharing the same `priorityGroup`. + +### 5. Elastic concurrency (`@RqueueListener.concurrency min < max`) + +Falls back to fixed `max` on NATS in v1. Implement auto-scaling poller count based on queue depth via `MessageBroker.size()`. + +### 6. `@RqueueHandler(primary)` on NATS + +Ignored in v1 with a single boot WARN. On NATS all handler methods are dispatched independently (one consumer per method). `primary` could select the default handler when message type is ambiguous. + +### 7. Spring Boot configuration metadata (source annotation processor) + +`rqueue.nats.auto-create-kv-buckets` and sibling properties appear in the built metadata JSON but only from the compiled artifact. Add `spring-boot-configuration-processor` to the starter's `annotationProcessor` deps so IDEs pick up descriptions and defaults without a pre-build step. + +### 8. `RqueueStringDao` javadoc + +Mark as Redis-internal in the interface javadoc so future contributors don't try to add a NATS impl. + +--- + +## Local verification commands + +```bash +./gradlew :rqueue-core:test :rqueue-redis:test :rqueue-web:test :rqueue-nats:test -DincludeTags=unit +./gradlew :rqueue-spring-boot-starter:test --tests "com.github.sonus21.rqueue.spring.boot.integration.NatsBackendEndToEndIT" +./gradlew :rqueue-nats:test -DincludeTags=nats +``` + +## Commit-rule reminder + +`CLAUDE.md` forbids `Co-Authored-By:` for any AI tool. Use `Assisted-By: Claude Code` only. diff --git a/nats-task.md b/nats-task.md new file mode 100644 index 000000000..c56a5acd6 --- /dev/null +++ b/nats-task.md @@ -0,0 +1,196 @@ +# NATS backend port — task tracker + +Snapshot of `nats-backend` branch progress and what's left to land. Kept here so a fresh session can resume cleanly. + +## What's done + +### Phase 1 — internal SPI in `rqueue-core` +- `com.github.sonus21.rqueue.core.spi` package with: + - `MessageBroker` interface — `enqueue / enqueueWithDelay / pop / ack / nack / moveExpired / peek / size / subscribe / publish / capabilities` and the `default` reactive overloads (`enqueueReactive`, `enqueueWithDelayReactive`). + - `Capabilities` record (`supportsDelayedEnqueue`, `supportsScheduledIntrospection`, `supportsCronJobs`, `usesPrimaryHandlerDispatch`). + - `MessageBrokerFactory` + `MessageBrokerLoader` (ServiceLoader). +- `RedisMessageBroker` thin delegate over the existing `RqueueMessageTemplate`. +- Public-API additions only — no removals: `setMessageBroker` / `getMessageBroker` on `RqueueMessageTemplateImpl`, `SimpleRqueueListenerContainerFactory`, `RqueueMessageListenerContainer`. `RqueueMessageTemplate` interface frozen. +- Existing 461+ `:rqueue-core:test` cases pass unchanged; 14 new `RedisMessageBrokerDelegationTest` cases lock the delegation contract. + +### Phase 2 — `rqueue-nats` module + `JetStreamMessageBroker` +- New module `rqueue-nats` (broker-impl only, no Spring/Boot deps; jnats 2.25.2 as `api`). Auto-config and `@Conditional` wiring live in `rqueue-spring-boot-starter` and `rqueue-spring`, gated by `@ConditionalOnClass(io.nats.client.JetStream.class)` + `@ConditionalOnProperty(rqueue.backend=nats)`. +- `JetStreamMessageBroker`: + - Builder API (`builder().connection(...).jetStream(...).management(...).config(...).build()`). + - `enqueue` → `js.publish(subject, headers, payload)` with `Nats-Msg-Id` header for dedup. + - `enqueueWithDelay` → throws `UnsupportedOperationException` (NATS v1 doesn't support arbitrary delay). + - `pop` → ensures stream + durable consumer, caches `JetStreamSubscription`, `sub.fetch(batch, wait)`, stashes raw `Message` in `inFlight` keyed by `RqueueMessage.id` for ack/nack lookup. + - `ack` / `nack(delayMs)` → `Message.ack()` / `Message.nakWithDelay(...)`. + - `peek` → ephemeral pull consumer with `AckPolicy.None`, fetch + unsubscribe (no perturbation of the durable's ack-pending state). + - `size` → `jsm.getStreamInfo(stream).getStreamState().getMsgCount()`. + - `subscribe` / `publish` → core NATS `Dispatcher`, returns `AutoCloseable` that calls `closeDispatcher`. + - Reactive overrides via `js.publishAsync(...)` wrapped in `Mono.fromFuture`. + - DLQ bridge: `installDeadLetterBridge(QueueDetail, consumerName)` subscribes to `$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>` and republishes exhausted messages to the DLQ subject. +- `RqueueNatsConfig` POJO + nested `StreamDefaults` / `ConsumerDefaults` for stream replication/storage/retention/dedup-window and consumer ack-wait/max-deliver/max-ack-pending. +- `NatsProvisioner` (in `rqueue-nats/.../internal/`) — idempotent `ensureStream`, `ensureConsumer`, `ensureDlqStream`. Logs WARN if existing config drifts from desired (doesn't mutate). +- `JetStreamMessageBrokerFactory` + `META-INF/services/com.github.sonus21.rqueue.core.spi.MessageBrokerFactory` for ServiceLoader discovery (`name() == "nats"`). +- `RqueueNatsException` (RuntimeException) wraps `IOException` / `JetStreamApiException` with stream/subject/consumer context in the message. +- 9 Docker-gated ITs in `rqueue-nats/src/test/`: `EnqueueAck`, `Retry+DLQ`, `CompetingConsumers`, `IndependentConsumers`, `Dedup`, `Peek`, `PubSub`, `ReactiveEnqueue`, `DelayThrows`. All pass against `nats:2.10-alpine -js` via Testcontainers. + +### Phase 3 — Spring/Boot wiring + listener-container branch +- `RqueueNatsAutoConfig` (Boot) registered via `META-INF/spring/...AutoConfiguration.imports`, gated `@ConditionalOnClass(JetStream.class) + @ConditionalOnProperty(rqueue.backend=nats)`. Provides `Connection`, `JetStream`, `JetStreamManagement`, `MessageBroker`, `RqueueQueueMetricsProvider` beans. +- `RqueueNatsListenerConfig` (non-Boot) activated by `NatsBackendCondition`. `@EnableRqueue(backend=NATS)` opts in via the new `Backend` enum. +- `RqueueListenerAutoConfig`'s default Redis broker uses `@ConditionalOnMissingBean(MessageBroker.class)`; the NATS bean wins when present. +- `@RqueueListener.consumerName()` attribute (additive). `ConsumerNameResolver` resolves: `consumerName` if set, else `"rqueue-" + queue + "-" + bean + "_" + method` with everything outside `[A-Za-z0-9_-]` collapsed to `_` (NATS durable-name constraint). +- `RqueueMessageHandler` skips primary-validation and logs one boot WARN listing `@RqueueHandler` annotated methods when capability says no primary dispatch. +- Cross-handler validation: `(queue, consumerName)` collisions fail boot fast. + +### Phase 3.5 — runtime path +- `BrokerMessagePoller` in `rqueue-core/listener/`. One thread per `(queue, consumerName, priority)` triple per `@RqueueListener.concurrency.max`. Loop: `broker.pop` → deserialize via `MessageConverter` → reflection-invoke the bound `HandlerMethod` (calls `createWithResolvedBean()` so bean-name lookup works) → `broker.ack` / `broker.nack(delayMs)` with `TaskExecutionBackOff`. +- `RqueueMessageListenerContainer` branches on `messageBroker != null && !capabilities.usesPrimaryHandlerDispatch()`: + - `startBrokerPollers()` enumerates active queues + handler methods, resolves consumer names, spawns pollers. + - `MessageScheduler` not started; `RqueueMessageHandler` primary loop bypassed. + - `doStop()` signals every poller; `doDestroy()` calls `broker.close()` if `AutoCloseable`. +- `BaseMessageSender.enqueue` routes through `MessageBroker.enqueue` when the active broker has `!usesPrimaryHandlerDispatch`. `storeMessageMetadata` short-circuits on the same flag. +- `RqueueListenerAutoConfig.rqueueMessageTemplate` propagates the autowired `MessageBroker` onto the template bean — without that, `BaseMessageSender#enqueue` would silently fall back to the Redis publish path. +- `SimpleRqueueListenerContainerFactory.createMessageListenerContainer()` skips the `redisConnectionFactory != null` assertion when a non-Redis broker is wired. + +### Phase 4 — dashboard + `QueueDetail` NATS fields +- `QueueDetail` adds nullable NATS fields with `resolved*` helpers: `natsStream`, `natsSubject`, `natsDlqStream`, `natsDlqSubject`, `natsAckWaitOverride`, `natsMaxDeliverOverride`, `natsDedupWindow`. Defaults derived from `queueName` when null. +- `RqueueQDetailServiceImpl` routes `size` / `peek` through `MessageBroker` when set; falls back to existing Redis path otherwise. +- `DataViewResponse` adds `hideScheduledPanel` / `hideCronJobs` flags. Pebble template `base.html` hides the "Scheduled" sidebar entry when the flag is set. +- Dashboard chain (`RqueueRestController`, `RqueueDashboardChartServiceImpl`, etc.) gated `@Conditional(RedisBackendCondition)`; on NATS the dashboard reports broker-derived sizes only. + +### Cross-phase summary +- Pluggable selection via `rqueue.backend=redis|nats` (default `redis`) and classpath presence. +- `RqueueConfig` carries the active `Backend` enum; downstream beans branch on that instead of probing the classpath. +- `Backend.AUTO` removed; `@EnableRqueue.backend()` defaults to `REDIS`. + +### Backend wiring split +- New module `rqueue-redis` with the Redis-shaped impls (DAOs, lock manager, KV-shaped beans). +- `Backend` enum + `RedisBackendCondition` / `NatsBackendCondition` in `rqueue-core`. +- `RqueueConfig.backend` field (default `REDIS`) bound from the `rqueue.backend` property; `Backend.AUTO` removed. +- `RqueueListenerBaseConfig.rqueueConfig(...)` factory tolerates a missing `RedisConnectionFactory`. +- `RqueueRedisTemplate` and `RqueueMessageTemplateImpl` constructors tolerate null Redis connection factory (NATS path constructs them for type satisfaction but never invokes Redis ops on them). +- `SimpleRqueueListenerContainerFactory` skips its `redisConnectionFactory != null` assertion when a non-Redis broker is set. +- `BaseMessageSender` routes producer enqueue through `MessageBroker.enqueue` when broker has `!usesPrimaryHandlerDispatch`; `storeMessageMetadata` short-circuits on the same flag. +- `RqueueQueueMetricsProvider` is the new backend-agnostic interface for queue-depth gauges; `RedisRqueueQueueMetricsProvider` (rqueue-redis) and `NatsRqueueQueueMetricsProvider` (rqueue-nats) supply impls. `RqueueMetrics` now reads through this provider, decoupled from `RqueueStringDao`. + +### NATS-native impls (KV-backed) +- `NatsRqueueLockManager` — KV bucket `rqueue-locks`, atomic create/release with revisioned delete, 6 ITs. +- `NatsRqueueSystemConfigDao` — KV bucket `rqueue-queue-config`, in-process cache, 6 ITs. +- `NatsRqueueJobDao` — KV bucket `rqueue-jobs`, scan-by-message-id, 7 ITs. +- `NatsRqueueMessageMetadataService` — KV bucket `rqueue-message-metadata`, 8 ITs. +- `NatsRqueueUtilityService` — admin-only stub returning "not supported" responses. +- `NatsRqueueStringDao` deleted — no consumer on the NATS path needs it. + +### Bean-graph cleanup +- `RqueueStringDao` consumers (`RqueueLockManagerImpl`, `RqueueJobDaoImpl`, `RqueueMessageMetadataServiceImpl`, `RqueueSystemManagerServiceImpl`, `RqueueUtilityServiceImpl`, `RqueueMetrics`'s old size getter) all gated `@Conditional(RedisBackendCondition.class)` or refactored to use `RqueueQueueMetricsProvider`. +- `RqueueStringDao` is now strictly internal to the Redis backend; no NATS-path bean autowires it. +- `BaseMessageSender`, `RqueueMessageManagerImpl`, `RqueueEndpointManagerImpl`, `RqueueBeanProvider` reverted to plain `@Autowired` (no more `required=false` shotgun) — every required interface has either a Redis impl or a `NatsRqueueXxx` impl. + +### CI +- `nats_integration_test` job in `.github/workflows/java-ci.yaml` installs nats-server v2.10.22 binary directly (no Docker), sets `NATS_RUNNING=true` + `NATS_URL`, mirrors the `redis_cluster_test` pattern. +- `AbstractNatsBootIT` and `AbstractJetStreamIT` honor `NATS_RUNNING` (CI path) and fall back to Testcontainers (local Docker path). +- Tests are tagged `@Tag("nats")` via `NatsIntegrationTest` / `NatsUnitTest` meta-annotations. + +### Module moves to `rqueue-redis` +- DAO impls: `RqueueStringDaoImpl`, `RqueueJobDaoImpl`, `RqueueMessageMetadataDaoImpl`, `RqueueQStatsDaoImpl`, `RqueueSystemConfigDaoImpl` → `rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/dao/`. +- Metrics: `RedisRqueueQueueMetricsProvider` → `rqueue-redis/.../redis/metrics/`. +- 5 `@Bean` factories from `RqueueListenerBaseConfig` → `RqueueRedisListenerConfig`: `rqueueRedisLongTemplate`, `rqueueRedisListenerContainerFactory`, `stringRqueueRedisTemplate`, `rqueueInternalPubSubChannel`, `rqueueStringDao`. +- 5 more `@Bean` factories: `scheduledMessageScheduler`, `processingMessageScheduler`, `rqueueWorkerRegistry`, `rqueueLockManager`, `rqueueQueueMetrics`. +- 6 service impls: `RqueueDashboardChartServiceImpl`, `RqueueJobServiceImpl`, `RqueueMessageMetadataServiceImpl`, `RqueueQDetailServiceImpl`, `RqueueSystemManagerServiceImpl`, `RqueueUtilityServiceImpl` → `rqueue-redis/.../redis/web/service/impl/`. + +### `rqueue-web` module extraction +- New module `rqueue-web` registered in `settings.gradle`. `rqueue-spring-boot-starter`, `rqueue-spring`, `rqueue-redis`, and `rqueue-spring-common-test` declare `api project(":rqueue-web")` so the dashboard ships by default; consumers `` it for headless workers. +- Moved out of `rqueue-core/web/...` into `rqueue-web/web/...`: 5 controllers (`Base`, `BaseReactive`, `RqueueRest`, `RqueueView`, `ReactiveRqueueRest`, `ReactiveRqueueView`), `RqueueWebExceptionAdvice`, `RqueueViewControllerServiceImpl`, and the 6 web-only service interfaces (`RqueueDashboardChartService`, `RqueueJobMetricsAggregatorService`, `RqueueJobService`, `RqueueQDetailService`, `RqueueSystemManagerService`, `RqueueViewControllerService`). Stayed in core: `RqueueMessageMetadataService` and `RqueueUtilityService` interfaces (consumed by listener / endpoint manager). +- Moved out of `rqueue-core/utils/pebble/`: 7 Pebble extension classes → `rqueue-web/utils/pebble/` (same package, no import changes). +- Moved out of `rqueue-core/src/main/resources/`: `templates/rqueue/**`, `public/rqueue/**` (CSS, JS, vendor assets) → `rqueue-web/src/main/resources/`. +- Moved out of `rqueue-core/src/test/`: `web/**` and `utils/pebble/**` test files → `rqueue-web/src/test/`. **Currently unbuildable** — see Pending. +- Pebble view-resolver `@Bean`s extracted from `RqueueListenerBaseConfig` into a new `RqueueWebViewConfig` in `rqueue-web/web/config/`. Picked up via the existing `com.github.sonus21.rqueue.web` component scan. +- `rqueue-core/build.gradle` dropped: `spring-webmvc`, `spring-webflux`, `jakarta.servlet-api`, `pebble-spring7`, `seruco/base62`, `hibernate-validator`, `org.glassfish:jakarta.el`. Added `reactor-core` directly (no longer comes via `spring-webflux`). `jakarta.validation-api` retained for DTO annotations. + +### `HttpUtils` JDK-client migration +- `HttpUtils.readUrl` rewritten to use `java.net.http.HttpClient` + Jackson; `org.springframework.web.client.RestTemplate` removed. `spring-web` dropped from `rqueue-core` deps. +- `joinPath` retained unchanged; `RqueueWebConfig.getUrlPrefix` still calls it. + +### Backend-agnostic worker registry +- New SPI in `rqueue-core`: `WorkerRegistryStore` (7 narrow KV-shaped methods). `RqueueWorkerRegistryImpl` relocated from `rqueue-redis/redis/worker/` to `rqueue-core/worker/`; takes `(RqueueConfig, WorkerRegistryStore)` in the constructor. All heartbeat scheduling / view assembly logic backend-neutral. +- `RedisWorkerRegistryStore` (rqueue-redis) wraps `RqueueRedisTemplate` over `set`/`get`/`mget`/hash ops. +- `NatsWorkerRegistryStore` (rqueue-nats) wraps two JetStream KV buckets: `rqueue-workers` (TTL = `workerRegistry.workerTtl`), `rqueue-worker-heartbeats` (TTL = `workerRegistry.queueTtl`). Hash-of-strings emulated as flattened keys `__`. `refreshQueueTtl` is a no-op since NATS resets per-entry age on each write. +- Wired in `RqueueRedisListenerConfig` and `RqueueNatsAutoConfig` under `@ConditionalOnMissingBean`. + +### NATS KV bucket lifecycle (`rqueue.nats.autoCreateKvBuckets`) +- New `com.github.sonus21.rqueue.nats.kv.NatsKvBuckets` constants class — single source of truth for the 6 bucket names (`QUEUE_CONFIG`, `JOBS`, `LOCKS`, `MESSAGE_METADATA`, `WORKERS`, `WORKER_HEARTBEATS`) + `ALL_BUCKETS` list. Every store / dao now references this constant instead of a private string. +- New `com.github.sonus21.rqueue.nats.kv.NatsKvBucketValidator` — config-source-agnostic class; constructor takes `(Connection, boolean autoCreate)`. Static `validate(Connection, boolean)` walks `ALL_BUCKETS` via `kvm.getStatus(name)` and aborts with `IllegalStateException` listing missing buckets. Implements `InitializingBean` so the bean form re-runs the same check. +- New `rqueue.nats.autoCreateKvBuckets` field on `RqueueNatsProperties` (default `true`). `rqueue-nats` itself never reads `rqueue.nats.*` keys directly — the property flows in only through the auto-config. +- Two enforcement layers in `RqueueNatsAutoConfig`: + 1. Inline call to `NatsKvBucketValidator.validate(connection, props.isAutoCreateKvBuckets())` inside the `natsConnection` `@Bean` factory, so validation completes during `Connection` bean creation (strictly before any other NATS bean can inject the connection). + 2. `@Bean public NatsKvBucketValidator natsKvBucketValidator(...)` declared from `RqueueNatsProperties`. Five NATS components (`NatsRqueueSystemConfigDao`, `NatsRqueueJobDao`, `NatsRqueueLockManager`, `NatsRqueueMessageMetadataService`, `NatsWorkerRegistryStore`) plus the `WorkerRegistryStore` `@Bean` factory carry `@DependsOn("natsKvBucketValidator")` so they wait on it even when the inline path is bypassed. + +### Web-layer infrastructure for capability-aware errors +- `BackendCapabilityException` (in `rqueue-core/exception/`) — carries `{backend, operation, reason}`. Mapped to HTTP 501 with structured JSON body by `RqueueWebExceptionAdvice` (in `rqueue-web/web/controller/`, scoped `@RestControllerAdvice(basePackageClasses = ...)`). No callers yet — landed as scaffolding for the upcoming web-service repository-interface refactor. + +### README — NATS backend section +- New "NATS backend" section in `README.md` covering: the 6 KV buckets (table with name, purpose, TTL behaviour, code link), how buckets are configured (lazy / immutable `ttl` / connection wiring), pre-create commands for restricted JetStream accounts, the `rqueue.nats.autoCreateKvBuckets=false` flag and its two-layer enforcement, and a recreate-with-new-TTL recipe. + +### CI & PR +- Branch `nats-backend` pushed to `origin`. ~50 commits, all carry `Assisted-By: Claude Code` only (no `Co-Authored-By:`). `CLAUDE.md` documents the rule. + +## Pending items + +### Build green — three test-compile failures across the tree + +Same root cause for two of three (cross-module visibility of test fixtures). Pick **option 1** below: promote the offenders to `rqueue-test-util/src/main`. + +1. **`rqueue-redis:compileTestJava`** — moved tests reference `CoreUnitTest` (annotation) and `QueueStatisticsTest` (fixture data) still living in `rqueue-core/src/test`. +2. **`rqueue-web:compileTestJava`** — `DateTimeFunctionTest`, `RqueueTaskMetricsAggregatorServiceTest`, `RqueuePebbleExtensionTest` reference `CoreUnitTest` and `TestUtils.createQueueDetail`, both still in `rqueue-core/src/test`. +3. **`rqueue-nats:compileTestJava`** — `JetStreamMessageBrokerDelayThrowsTest:36` calls `new JetStreamMessageBroker(Connection, JetStream, JetStreamManagement, RqueueNatsConfig, ObjectMapper)` from outside the broker's package. The constructor is package-private (regression from a recent visibility tighten). Fix: widen the constructor to `public`, or use the existing `JetStreamMessageBroker.builder()` API in the test. + +Plan for items 1 + 2: +- Move `rqueue-core/src/test/java/com/github/sonus21/rqueue/CoreUnitTest.java` → `rqueue-test-util/src/main/java/com/github/sonus21/rqueue/CoreUnitTest.java`. +- Move `rqueue-core/src/test/java/com/github/sonus21/rqueue/utils/TestUtils.java` → `rqueue-test-util/src/main/java/com/github/sonus21/rqueue/utils/TestUtils.java`. +- (Optional) `QueueStatisticsTest` fixture data — promote helpers if the moved Redis tests still need them. +- All consumer modules already pull `rqueue-test-util` as `testImplementation`, so no build wiring needed. + +Then re-run: +``` +./gradlew :rqueue-core:test :rqueue-redis:test :rqueue-web:test :rqueue-nats:test -DincludeTags=unit +./gradlew :rqueue-spring-boot-starter:test --tests "com.github.sonus21.rqueue.spring.boot.integration.NatsBackendEndToEndIT" +``` + +### Web-layer NATS dashboard gap (new follow-up) + +All 4 controllers and the 5 web service impls (`RqueueDashboardChartService*`, `RqueueQDetailService*`, `RqueueJobService*`, `RqueueSystemManagerService*`, `RqueueUtilityService*`) are still gated `@Conditional(RedisBackendCondition)`. On NATS the dashboard reports broker-derived sizes only; no charts, no message browse, no admin ops. Plan to fix: + +1. Introduce repository interfaces in `rqueue-core/repository/` for the few storage primitives the web services share (queue browsing, time-series counters, atomic move). Web service impls move into core / `rqueue-web` and depend only on the repos. +2. Redis impls of the repos stay in `rqueue-redis`; NATS impls go in `rqueue-nats` and throw `BackendCapabilityException("nats", "operation", "reason")` for primitives JetStream can't model (positional message moves, time-bucket charts). +3. Drop `@Conditional(RedisBackendCondition)` from controllers; the advice already maps the exception to HTTP 501 with a structured body. +4. Extend the existing `Capabilities` record (`MessageBroker.capabilities()`) with dashboard-op flags so the front-end can hide unsupported panels instead of relying on 501s. Expose `GET /rqueue/api/capabilities`. + +Order of operations: easiest first — `RqueueSystemManagerService` (already mostly goes through `RqueueSystemConfigDao`), then `RqueueJobService`, `RqueueViewControllerService`, then `RqueueQDetailService` (needs new `MessageBrowsingRepository`), then `RqueueDashboardChartService` and `RqueueUtilityService.move/enqueue` last (these throw on NATS). + +### Spring Boot configuration metadata + +`spring-configuration-metadata.json` has no entry for `rqueue.nats.autoCreateKvBuckets`. IDE autocomplete won't show it. Easy follow-up: add `spring-boot-configuration-processor` to the starter's annotation processors if not already wired. + +### Other open follow-ups + +- **`RqueueStringDao` interface** — keep Redis-only; document as Redis-internal in javadoc. +- **`RqueueMessageMetadataDao`, `RqueueQStatsDao`** — no NATS impls needed; all consumers are Redis-only gated. Re-verify in light of the web-layer refactor above. +- **Reactive listener container** — only enqueue side is reactive in v1. Phase 5 territory. +- **Delayed/scheduled/cron messages on NATS** — throws `UnsupportedOperationException`. Out of scope for v1. +- **Cross-queue `priorityGroup` weighting on NATS** — boot WARN, not honored. Acceptable for v1. +- **Elastic `@RqueueListener.concurrency` (min < max)** — falls back to fixed `max` on NATS. Acceptable. +- **`@RqueueHandler(primary)` on NATS** — ignored, single boot WARN. +- **PR open on `sonus21/rqueue:nats-backend`** — branch pushed; user opened the PR through the GitHub UI. + +## Local verification commands + +``` +./gradlew :rqueue-core:test :rqueue-redis:test :rqueue-nats:test -DincludeTags=unit +./gradlew :rqueue-spring-boot-starter:test --tests "com.github.sonus21.rqueue.spring.boot.integration.NatsBackendEndToEndIT" +./gradlew :rqueue-nats:test --tests "com.github.sonus21.rqueue.nats.lock.NatsRqueueLockManagerIT" +./gradlew :rqueue-nats:test --tests "com.github.sonus21.rqueue.nats.dao.NatsRqueueSystemConfigDaoIT" +./gradlew :rqueue-nats:test --tests "com.github.sonus21.rqueue.nats.dao.NatsRqueueJobDaoIT" +./gradlew :rqueue-nats:test --tests "com.github.sonus21.rqueue.nats.service.NatsRqueueMessageMetadataServiceIT" +``` + +## Commit-rule reminder + +`CLAUDE.md` at the repo root forbids `Co-Authored-By:` for any AI tool. Use `Assisted-By: Claude Code` as a single trailer per commit. The trailer rewrite has already been applied to historical commits; new commits just need the right form. diff --git a/rqueue-core/build.gradle b/rqueue-core/build.gradle index ff0a78c68..b941f9c14 100644 --- a/rqueue-core/build.gradle +++ b/rqueue-core/build.gradle @@ -43,24 +43,17 @@ dependencies { api "tools.jackson.core:jackson-core:${jacksonVersion}" api "tools.jackson.core:jackson-databind:${jacksonVersion}" api "com.fasterxml.jackson.core:jackson-annotations:${jacksonAnnotationsVersion}" - // https://mvnrepository.com/artifact/jakarta.servlet/jakarta.servlet-api - api "jakarta.servlet:jakarta.servlet-api:${jakartaServletVersion}" - // https://mvnrepository.com/artifact/jakarta.validation/jakarta.validation-api + // jakarta.validation annotations are still used by request DTOs in core (consumed by + // RqueueUtilityService, which the core engine depends on for admin operations). api "jakarta.validation:jakarta.validation-api:${jakartaValidationApiVersion}" - implementation 'jakarta.el:jakarta.el-api:5.0.0' - runtimeOnly 'org.glassfish:jakarta.el:4.0.2' - // https://mvnrepository.com/artifact/org.springframework/spring-webmvc - api "org.springframework:spring-webmvc:${springVersion}" - api "org.springframework:spring-webflux:${springVersion}" - api "io.pebbletemplates:pebble-spring7:${pebbleVersion}" - api "io.seruco.encoding:base62:${serucoEncodingVersion}" + // Mono / Flux types appear in core service interfaces (reactive variants of admin ops). + api "io.projectreactor:reactor-core:${projectReactorReactorTestVersion}" // https://mvnrepository.com/artifact/org.apache.commons/commons-collections4 api "org.apache.commons:commons-collections4:${apacheCommonCollectionVerion}" - // https://mvnrepository.com/artifact/org.hibernate.validator/hibernate-validator - api "org.hibernate.validator:hibernate-validator:${hibernateValidatorVersion}" // https://mvnrepository.com/artifact/io.micrometer/micrometer-core api "io.micrometer:micrometer-core:${microMeterVersion}" testImplementation "io.lettuce:lettuce-core:${lettuceVersion}" + testImplementation "io.projectreactor:reactor-test:${projectReactorReactorTestVersion}" testImplementation project(":rqueue-test-util") } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/annotation/RqueueListener.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/annotation/RqueueListener.java index fc799d9ff..6e2bd2df2 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/annotation/RqueueListener.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/annotation/RqueueListener.java @@ -16,6 +16,7 @@ package com.github.sonus21.rqueue.annotation; +import com.github.sonus21.rqueue.enums.QueueType; import com.github.sonus21.rqueue.utils.Constants; import java.lang.annotation.Documented; import java.lang.annotation.ElementType; @@ -191,4 +192,36 @@ * @return exceptions those will not be retried */ Class[] doNotRetry() default {}; + + /** + * Optional explicit consumer name for backends that require per-listener durable identity + * (currently the NATS / JetStream backend uses this as the durable consumer name). + * + *

When empty (the default), the listener container synthesizes a name of the form + * {@code "rqueue--#"}. The Redis backend ignores this attribute; + * it's strictly additive and exists so multiple {@link RqueueListener} methods on the same + * NATS-backed queue can be told apart by JetStream. + * + * @return durable consumer name override; empty for "auto-generate" + */ + String consumerName() default ""; + + /** + * Delivery mode for this queue. + * + *

    + *
  • {@link QueueType#QUEUE} (default) — competing-consumer semantics: each + * message is delivered to exactly one listener instance. Maps to a JetStream + * {@code WorkQueue} stream. Multiple listeners share the load. + *
  • {@link QueueType#STREAM} — fan-out semantics: every independent listener + * group receives a copy of each message. Maps to a JetStream {@code Limits} stream. + * Each listener must use a distinct {@link #consumerName()} so JetStream tracks its + * own position independently. + *
+ * + *

The Redis backend ignores this attribute (Redis queue names already encode the routing). + * + * @return queue delivery mode + */ + QueueType mode() default QueueType.QUEUE; } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/common/RqueueRedisTemplate.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/common/RqueueRedisTemplate.java index 91329fabb..2028d8ad3 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/common/RqueueRedisTemplate.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/common/RqueueRedisTemplate.java @@ -38,6 +38,13 @@ public class RqueueRedisTemplate { protected RedisTemplate redisTemplate; public RqueueRedisTemplate(RedisConnectionFactory redisConnectionFactory) { + if (redisConnectionFactory == null) { + // Permitted on the NATS backend path, where the template is constructed for type + // satisfaction but never used for Redis operations. Leaving redisTemplate null fails + // fast and obviously if anyone does try to use it. + this.redisTemplate = null; + return; + } this.redisTemplate = RedisUtils.getRedisTemplate(redisConnectionFactory); this.redisTemplate.afterPropertiesSet(); } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/Backend.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/Backend.java new file mode 100644 index 000000000..aa43c4e23 --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/Backend.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2019-2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ + +package com.github.sonus21.rqueue.config; + +/** + * The active rqueue backend. Bound from the {@code rqueue.backend} configuration property + * (Spring binds the string value case-insensitively to the matching enum constant); defaults to + * {@link #REDIS}. + * + *

The {@code RqueueConfig#getBackend()} method exposes the resolved value so any bean that + * wants to branch on the active backend can check it directly instead of relying on classpath + * probes for {@code RedisConnectionFactory} or {@code JetStream}. + */ +public enum Backend { + REDIS, + NATS +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/NatsBackendCondition.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/NatsBackendCondition.java new file mode 100644 index 000000000..36cee1256 --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/NatsBackendCondition.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ + +package com.github.sonus21.rqueue.config; + +import org.springframework.context.annotation.Condition; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.core.type.AnnotatedTypeMetadata; + +/** + * Spring {@link Condition} that matches when the active rqueue backend is {@link Backend#NATS}. + * Companion to {@link RedisBackendCondition}; together they ensure every Redis-shaped interface + * gets exactly one bean (Redis impl when backend=redis, no-op stub when backend=nats) so + * consumers don't need {@code @Autowired(required = false)} or null guards. + */ +public class NatsBackendCondition implements Condition { + @Override + public boolean matches(ConditionContext ctx, AnnotatedTypeMetadata md) { + return RedisBackendCondition.resolveBackend(ctx) == Backend.NATS; + } +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RedisBackendCondition.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RedisBackendCondition.java new file mode 100644 index 000000000..bbebbbcf9 --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RedisBackendCondition.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ + +package com.github.sonus21.rqueue.config; + +import org.springframework.context.annotation.Condition; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.core.type.AnnotatedTypeMetadata; + +/** + * Spring {@link Condition} that matches when the active rqueue backend is {@link Backend#REDIS}. + * Reads the {@code rqueue.backend} property; absent or unparseable values default to REDIS so the + * existing behavior is preserved. + */ +public class RedisBackendCondition implements Condition { + @Override + public boolean matches(ConditionContext ctx, AnnotatedTypeMetadata md) { + return resolveBackend(ctx) == Backend.REDIS; + } + + static Backend resolveBackend(ConditionContext ctx) { + String raw = ctx.getEnvironment().getProperty("rqueue.backend"); + if (raw == null || raw.isBlank()) { + return Backend.REDIS; + } + try { + return Backend.valueOf(raw.trim().toUpperCase()); + } catch (IllegalArgumentException ex) { + return Backend.REDIS; + } + } +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RqueueConfig.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RqueueConfig.java index ea3dcef5d..9007c3dd7 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RqueueConfig.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RqueueConfig.java @@ -44,7 +44,17 @@ public class RqueueConfig { @Getter - private static final String brokerId = UUID.randomUUID().toString(); + private static final String brokerId = generateBrokerId(); + + private static String generateBrokerId() { + String chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + java.util.Random random = new java.util.Random(); + StringBuilder sb = new StringBuilder(8); + for (int i = 0; i < 8; i++) { + sb.append(chars.charAt(random.nextInt(chars.length()))); + } + return sb.toString(); + } private static final AtomicLong counter = new AtomicLong(1); private final RedisConnectionFactory connectionFactory; @@ -52,6 +62,15 @@ public class RqueueConfig { private final boolean sharedConnection; private final int dbVersion; + /** + * Active rqueue backend. Defaults to {@link Backend#REDIS} so every existing call site that + * constructs {@code RqueueConfig} via the Lombok-generated constructor keeps the same + * behavior. The {@code rqueueConfig} bean factory in {@link RqueueListenerBaseConfig} reads the + * {@code rqueue.backend} property and overrides it. Beans that need to know the active backend + * should call {@link #getBackend()} instead of probing the classpath. + */ + private Backend backend = Backend.REDIS; + @Value("${rqueue.reactive.enabled:false}") private boolean reactiveEnabled; diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RqueueListenerBaseConfig.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RqueueListenerBaseConfig.java index 3f6ee0350..2bd43887a 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RqueueListenerBaseConfig.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/RqueueListenerBaseConfig.java @@ -16,46 +16,27 @@ package com.github.sonus21.rqueue.config; -import static com.github.sonus21.rqueue.utils.RedisUtils.getRedisTemplate; - -import com.github.sonus21.rqueue.common.RqueueLockManager; import com.github.sonus21.rqueue.common.RqueueRedisTemplate; -import com.github.sonus21.rqueue.common.impl.RqueueLockManagerImpl; import com.github.sonus21.rqueue.converter.MessageConverterProvider; -import com.github.sonus21.rqueue.core.ProcessingQueueMessageScheduler; import com.github.sonus21.rqueue.core.RqueueBeanProvider; -import com.github.sonus21.rqueue.core.RqueueInternalPubSubChannel; import com.github.sonus21.rqueue.core.RqueueMessageIdGenerator; import com.github.sonus21.rqueue.core.RqueueMessageTemplate; -import com.github.sonus21.rqueue.core.RqueueRedisListenerContainerFactory; -import com.github.sonus21.rqueue.core.ScheduledQueueMessageScheduler; import com.github.sonus21.rqueue.core.impl.RqueueMessageTemplateImpl; import com.github.sonus21.rqueue.core.impl.UuidV4RqueueMessageIdGenerator; -import com.github.sonus21.rqueue.dao.RqueueStringDao; -import com.github.sonus21.rqueue.dao.impl.RqueueStringDaoImpl; -import com.github.sonus21.rqueue.listener.RqueueMessageListenerContainer; -import com.github.sonus21.rqueue.metrics.RqueueQueueMetrics; +import com.github.sonus21.rqueue.serdes.RqueueSerDes; +import com.github.sonus21.rqueue.serdes.RqueueTypeFactory; +import com.github.sonus21.rqueue.serdes.SerializationUtils; import com.github.sonus21.rqueue.utils.RedisUtils; import com.github.sonus21.rqueue.utils.condition.MissingRqueueMessageIdGenerator; -import com.github.sonus21.rqueue.utils.condition.ReactiveEnabled; -import com.github.sonus21.rqueue.utils.pebble.ResourceLoader; -import com.github.sonus21.rqueue.utils.pebble.RqueuePebbleExtension; -import com.github.sonus21.rqueue.worker.RqueueWorkerRegistry; -import com.github.sonus21.rqueue.worker.RqueueWorkerRegistryImpl; -import io.pebbletemplates.pebble.PebbleEngine; -import io.pebbletemplates.spring.extension.SpringExtension; -import io.pebbletemplates.spring.reactive.PebbleReactiveViewResolver; -import io.pebbletemplates.spring.servlet.PebbleViewResolver; +import com.github.sonus21.rqueue.utils.condition.MissingRqueueSerDes; +import com.github.sonus21.rqueue.utils.condition.MissingRqueueTypeFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Conditional; import org.springframework.data.redis.connection.ReactiveRedisConnectionFactory; import org.springframework.data.redis.connection.RedisConnectionFactory; -import org.springframework.data.redis.core.RedisTemplate; -import org.springframework.web.servlet.ViewResolver; /** * This is a base configuration class for Rqueue, that is used in Spring and Spring boot Rqueue libs @@ -71,8 +52,6 @@ public abstract class RqueueListenerBaseConfig { public static final int MAX_DB_VERSION = 2; - private static final String TEMPLATE_DIR = "templates/rqueue/"; - private static final String TEMPLATE_SUFFIX = ".html"; @Autowired(required = false) protected final SimpleRqueueListenerContainerFactory simpleRqueueListenerContainerFactory = @@ -115,40 +94,62 @@ protected MessageConverterProvider getMessageConverterProvider() { * @param dbVersion database version * @return {@link RedisConnectionFactory} object. */ + /** + * Backwards-compatible overload preserved for callers (notably existing tests) that constructed + * an {@code RqueueConfig} without specifying a backend. Delegates with {@link Backend#REDIS}. + */ + public RqueueConfig rqueueConfig( + ConfigurableBeanFactory beanFactory, String versionKey, Integer dbVersion) { + return rqueueConfig(beanFactory, Backend.REDIS, versionKey, dbVersion); + } + @Bean public RqueueConfig rqueueConfig( ConfigurableBeanFactory beanFactory, + @Value("${rqueue.backend:REDIS}") Backend backend, @Value("${rqueue.version.key:__rq::version}") String versionKey, @Value("${rqueue.db.version:}") Integer dbVersion) { boolean sharedConnection = false; - if (simpleRqueueListenerContainerFactory.getRedisConnectionFactory() == null) { - sharedConnection = true; - simpleRqueueListenerContainerFactory.setRedisConnectionFactory( - beanFactory.getBean(RedisConnectionFactory.class)); - } - if (reactiveEnabled - && simpleRqueueListenerContainerFactory.getReactiveRedisConnectionFactory() == null) { - sharedConnection = true; - simpleRqueueListenerContainerFactory.setReactiveRedisConnectionFactory( - beanFactory.getBean(ReactiveRedisConnectionFactory.class)); - } RedisConnectionFactory connectionFactory = simpleRqueueListenerContainerFactory.getRedisConnectionFactory(); - RqueueRedisTemplate rqueueRedisTemplate = new RqueueRedisTemplate<>(connectionFactory); + if (backend == Backend.REDIS) { + if (connectionFactory == null) { + sharedConnection = true; + connectionFactory = beanFactory.getBean(RedisConnectionFactory.class); + simpleRqueueListenerContainerFactory.setRedisConnectionFactory(connectionFactory); + } + if (reactiveEnabled + && simpleRqueueListenerContainerFactory.getReactiveRedisConnectionFactory() == null) { + sharedConnection = true; + simpleRqueueListenerContainerFactory.setReactiveRedisConnectionFactory( + beanFactory.getBean(ReactiveRedisConnectionFactory.class)); + } + } int version; - if (dbVersion == null) { - version = RedisUtils.updateAndGetVersion(rqueueRedisTemplate, versionKey, MAX_DB_VERSION); - } else if (dbVersion >= 1 && dbVersion <= MAX_DB_VERSION) { - RedisUtils.setVersion(rqueueRedisTemplate, versionKey, dbVersion); - version = dbVersion; + if (backend == Backend.REDIS) { + RqueueRedisTemplate rqueueRedisTemplate = + new RqueueRedisTemplate<>(connectionFactory); + if (dbVersion == null) { + version = RedisUtils.updateAndGetVersion(rqueueRedisTemplate, versionKey, MAX_DB_VERSION); + } else if (dbVersion >= 1 && dbVersion <= MAX_DB_VERSION) { + RedisUtils.setVersion(rqueueRedisTemplate, versionKey, dbVersion); + version = dbVersion; + } else { + throw new IllegalStateException("Rqueue db version '" + dbVersion + "' is not correct"); + } } else { - throw new IllegalStateException("Rqueue db version '" + dbVersion + "' is not correct"); + // Non-Redis backend (e.g. NATS): the on-Redis db-version negotiation does not apply. + version = (dbVersion != null && dbVersion >= 1 && dbVersion <= MAX_DB_VERSION) + ? dbVersion + : MAX_DB_VERSION; } - return new RqueueConfig( + RqueueConfig config = new RqueueConfig( connectionFactory, simpleRqueueListenerContainerFactory.getReactiveRedisConnectionFactory(), sharedConnection, version); + config.setBackend(backend); + return config; } @Bean @@ -185,108 +186,19 @@ protected RqueueMessageTemplate getMessageTemplate(RqueueConfig rqueueConfig) { } @Bean - public RedisTemplate rqueueRedisLongTemplate(RqueueConfig rqueueConfig) { - return getRedisTemplate(rqueueConfig.getConnectionFactory()); - } - - @Bean - public RqueueRedisListenerContainerFactory rqueueRedisListenerContainerFactory() { - return new RqueueRedisListenerContainerFactory(); - } - - /** - * This scheduler is used to pull messages from a scheduled queue to their respective queue. - * Internally it moves messages from ZSET to LIST based on the priority and current time. - * - * @return {@link ScheduledQueueMessageScheduler} object - */ - @Bean - public ScheduledQueueMessageScheduler scheduledMessageScheduler() { - return new ScheduledQueueMessageScheduler(); - } - - /** - * This scheduler is used to pull messages from processing queue to their respective queue. - * Internally it moves messages from ZSET to LIST based on the priority and current time. - * - * @return {@link ProcessingQueueMessageScheduler} object - */ - @Bean - public ProcessingQueueMessageScheduler processingMessageScheduler() { - return new ProcessingQueueMessageScheduler(); - } - - @Bean - public RqueueRedisTemplate stringRqueueRedisTemplate(RqueueConfig rqueueConfig) { - return new RqueueRedisTemplate<>(rqueueConfig.getConnectionFactory()); + @Conditional(MissingRqueueSerDes.class) + public RqueueSerDes rqueueSerDes() { + return SerializationUtils.getSerDes(); } @Bean - public RqueueStringDao rqueueStringDao(RqueueConfig rqueueConfig) { - return new RqueueStringDaoImpl(rqueueConfig); - } - - @Bean - public RqueueWorkerRegistry rqueueWorkerRegistry(RqueueConfig rqueueConfig) { - return new RqueueWorkerRegistryImpl(rqueueConfig); - } - - @Bean - public RqueueLockManager rqueueLockManager(RqueueStringDao rqueueStringDao) { - return new RqueueLockManagerImpl(rqueueStringDao); - } - - private PebbleEngine createPebbleEngine() { - ResourceLoader loader = new ResourceLoader(); - loader.setPrefix(TEMPLATE_DIR); - loader.setSuffix(TEMPLATE_SUFFIX); - return new PebbleEngine.Builder() - .extension(new RqueuePebbleExtension(), new SpringExtension(null)) - .loader(loader) - .build(); - } - - @Bean - public ViewResolver rqueueViewResolver() { - PebbleViewResolver resolver = new PebbleViewResolver(createPebbleEngine()); - resolver.setPrefix(TEMPLATE_DIR); - resolver.setSuffix(TEMPLATE_SUFFIX); - return resolver; - } - - @Bean - @Conditional(ReactiveEnabled.class) - public org.springframework.web.reactive.result.view.ViewResolver reactiveRqueueViewResolver() { - PebbleReactiveViewResolver resolver = new PebbleReactiveViewResolver(createPebbleEngine()); - resolver.setPrefix(TEMPLATE_DIR); - resolver.setSuffix(TEMPLATE_SUFFIX); - return resolver; - } - - @Bean - public RqueueQueueMetrics rqueueQueueMetrics( - RqueueRedisTemplate stringRqueueRedisTemplate) { - return new RqueueQueueMetrics(stringRqueueRedisTemplate); + @Conditional(MissingRqueueTypeFactory.class) + public RqueueTypeFactory rqueueTypeFactory() { + return SerializationUtils.getTypeFactory(); } @Bean public RqueueBeanProvider rqueueBeanProvider() { return new RqueueBeanProvider(); } - - @Bean - public RqueueInternalPubSubChannel rqueueInternalPubSubChannel( - RqueueRedisListenerContainerFactory rqueueRedisListenerContainerFactory, - RqueueMessageListenerContainer rqueueMessageListenerContainer, - RqueueConfig rqueueConfig, - RqueueBeanProvider rqueueBeanProvider, - @Qualifier("stringRqueueRedisTemplate") - RqueueRedisTemplate stringRqueueRedisTemplate) { - return new RqueueInternalPubSubChannel( - rqueueRedisListenerContainerFactory, - rqueueMessageListenerContainer, - rqueueConfig, - stringRqueueRedisTemplate, - rqueueBeanProvider); - } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/SimpleRqueueListenerContainerFactory.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/SimpleRqueueListenerContainerFactory.java index 9d214e5d8..c9426dd61 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/SimpleRqueueListenerContainerFactory.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/config/SimpleRqueueListenerContainerFactory.java @@ -26,6 +26,8 @@ import com.github.sonus21.rqueue.core.RqueueMessageTemplate; import com.github.sonus21.rqueue.core.impl.RqueueMessageTemplateImpl; import com.github.sonus21.rqueue.core.middleware.Middleware; +import com.github.sonus21.rqueue.core.spi.MessageBroker; +import com.github.sonus21.rqueue.core.spi.redis.RedisMessageBroker; import com.github.sonus21.rqueue.core.support.MessageProcessor; import com.github.sonus21.rqueue.listener.HardStrictPriorityPollerProperties; import com.github.sonus21.rqueue.listener.RqueueMessageHandler; @@ -91,6 +93,10 @@ public class SimpleRqueueListenerContainerFactory { // changed, same header is used in serialized and deserialization process. private MessageHeaders messageHeaders; + // Optional pluggable message broker (SPI). Additive: when null, behavior is unchanged and the + // factory uses the existing Redis-backed paths. + private MessageBroker messageBroker; + // Set priority mode for the pollers private PriorityMode priorityMode = PriorityMode.WEIGHTED; // Set HardStrictPriorityPollerProperties for HARD_STRICT priority mode poller @@ -303,7 +309,19 @@ public void setRqueueMessageTemplate(RqueueMessageTemplate messageTemplate) { * @return an object of {@link RqueueMessageListenerContainer} object */ public RqueueMessageListenerContainer createMessageListenerContainer() { - notNull(redisConnectionFactory, "redisConnectionFactory must not be null"); + if (messageBroker != null + && redisConnectionFactory != null + && !(messageBroker instanceof RedisMessageBroker)) { + throw new IllegalStateException( + "Both redisConnectionFactory and a non-Redis MessageBroker are configured. " + + "Configure exactly one transport: either set redisConnectionFactory for Redis, " + + "or set messageBroker for an alternative backend (e.g. NATS)."); + } + boolean nonRedisBroker = + messageBroker != null && !(messageBroker instanceof RedisMessageBroker); + if (!nonRedisBroker) { + notNull(redisConnectionFactory, "redisConnectionFactory must not be null"); + } notNull(messageConverterProvider, "messageConverterProvider must not be null"); if (rqueueMessageTemplate == null) { rqueueMessageTemplate = new RqueueMessageTemplateImpl( @@ -311,6 +329,9 @@ public RqueueMessageListenerContainer createMessageListenerContainer() { } RqueueMessageListenerContainer messageListenerContainer = new RqueueMessageListenerContainer( getRqueueMessageHandler(messageConverterProvider), rqueueMessageTemplate); + if (messageBroker != null) { + messageListenerContainer.setMessageBroker(messageBroker); + } messageListenerContainer.setAutoStartup(autoStartup); if (taskExecutor != null) { messageListenerContainer.setTaskExecutor(taskExecutor); @@ -561,4 +582,25 @@ public void setReactiveRedisConnectionFactory( notNull(reactiveRedisConnectionFactory, "reactiveRedisConnectionFactory can not be null"); this.reactiveRedisConnectionFactory = reactiveRedisConnectionFactory; } + + /** + * @return configured {@link MessageBroker} or {@code null} if none has been set + */ + public MessageBroker getMessageBroker() { + return messageBroker; + } + + /** + * Set the {@link MessageBroker} SPI instance to be used by the listener container. Additive: if + * not set, the container falls back to the existing Redis-backed code path. + * + *

If {@code messageBroker} is non-null and {@code redisConnectionFactory} is also set, the + * broker must be a {@link RedisMessageBroker}; otherwise {@link #createMessageListenerContainer()} + * will throw {@link IllegalStateException}. + * + * @param messageBroker the broker to use, or {@code null} to clear + */ + public void setMessageBroker(MessageBroker messageBroker) { + this.messageBroker = messageBroker; + } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/GenericMessageConverter.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/GenericMessageConverter.java index 4d77eefc0..f13bd5e99 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/GenericMessageConverter.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/GenericMessageConverter.java @@ -18,9 +18,15 @@ import static org.springframework.util.Assert.notNull; -import com.github.sonus21.rqueue.utils.SerializationUtils; +import com.github.sonus21.rqueue.serdes.RqJacksonSerDes; +import com.github.sonus21.rqueue.serdes.RqJacksonTypeFactory; +import com.github.sonus21.rqueue.serdes.RqueueSerDes; +import com.github.sonus21.rqueue.serdes.RqueueTypeFactory; +import com.github.sonus21.rqueue.serdes.SerializationUtils; +import com.github.sonus21.rqueue.serdes.TypeEnvelop; import java.lang.reflect.Field; import java.lang.reflect.TypeVariable; +import java.nio.charset.StandardCharsets; import java.util.Collection; import java.util.List; import lombok.AllArgsConstructor; @@ -33,8 +39,15 @@ import org.springframework.messaging.converter.SmartMessageConverter; import org.springframework.messaging.support.GenericMessage; import tools.jackson.core.JacksonException; -import tools.jackson.databind.JavaType; +import tools.jackson.core.JsonGenerator; +import tools.jackson.core.JsonParser; +import tools.jackson.databind.DeserializationContext; import tools.jackson.databind.ObjectMapper; +import tools.jackson.databind.SerializationContext; +import tools.jackson.databind.annotation.JsonDeserialize; +import tools.jackson.databind.annotation.JsonSerialize; +import tools.jackson.databind.deser.std.StdDeserializer; +import tools.jackson.databind.ser.std.StdSerializer; /** * A converter to turn the payload of a {@link Message} from serialized form to a typed String and @@ -47,13 +60,20 @@ public class GenericMessageConverter implements SmartMessageConverter { private final SmartMessageSerDes smartMessageSerDes; public GenericMessageConverter() { - ObjectMapper mapper = SerializationUtils.createObjectMapper(); - this.smartMessageSerDes = new SmartMessageSerDes(mapper); + this.smartMessageSerDes = + new SmartMessageSerDes(SerializationUtils.getSerDes(), SerializationUtils.getTypeFactory()); } public GenericMessageConverter(ObjectMapper objectMapper) { notNull(objectMapper, "objectMapper cannot be null"); - this.smartMessageSerDes = new SmartMessageSerDes(objectMapper); + this.smartMessageSerDes = new SmartMessageSerDes( + new RqJacksonSerDes(objectMapper), new RqJacksonTypeFactory(objectMapper)); + } + + public GenericMessageConverter(RqueueSerDes serDes, RqueueTypeFactory typeFactory) { + notNull(serDes, "serDes cannot be null"); + notNull(typeFactory, "typeFactory cannot be null"); + this.smartMessageSerDes = new SmartMessageSerDes(serDes, typeFactory); } /** @@ -117,16 +137,50 @@ public Object fromMessage(Message message, Class targetClass) { @AllArgsConstructor private static class Msg { - private String msg; + @JsonSerialize(using = Utf8BytesSerializer.class) + @JsonDeserialize(using = Utf8BytesDeserializer.class) + private byte[] msg; + private String name; } + private static class Utf8BytesSerializer extends StdSerializer { + + Utf8BytesSerializer() { + super(byte[].class); + } + + @Override + public void serialize(byte[] value, JsonGenerator gen, SerializationContext ctx) + throws JacksonException { + gen.writeString(new String(value, StandardCharsets.UTF_8)); + } + } + + private static class Utf8BytesDeserializer extends StdDeserializer { + + Utf8BytesDeserializer() { + super(byte[].class); + } + + @Override + public byte[] deserialize(JsonParser p, DeserializationContext ctx) throws JacksonException { + String text = p.getString(); + if (text == null) { + return null; + } + return text.getBytes(StandardCharsets.UTF_8); + } + } + public static class SmartMessageSerDes { - private final ObjectMapper objectMapper; + private final RqueueSerDes serDes; + private final RqueueTypeFactory typeFactory; - public SmartMessageSerDes(ObjectMapper objectMapper) { - this.objectMapper = objectMapper; + public SmartMessageSerDes(RqueueSerDes serDes, RqueueTypeFactory typeFactory) { + this.serDes = serDes; + this.typeFactory = typeFactory; } private String[] splitClassNames(String name) { @@ -197,11 +251,11 @@ private String getClassName(Object payload) { return getGenericFieldBasedClassName(payloadClass, payload); } - private JavaType getTargetType(Msg msg) throws ClassNotFoundException { + private TypeEnvelop getTargetType(Msg msg) throws ClassNotFoundException { String[] classNames = splitClassNames(msg.getName()); if (classNames.length == 1) { Class c = Thread.currentThread().getContextClassLoader().loadClass(msg.getName()); - return objectMapper.getTypeFactory().constructType(c); + return typeFactory.create(c); } Class envelopeClass = Thread.currentThread().getContextClassLoader().loadClass(classNames[0]); @@ -209,15 +263,15 @@ private JavaType getTargetType(Msg msg) throws ClassNotFoundException { for (int i = 1; i < classNames.length; i++) { classes[i - 1] = Thread.currentThread().getContextClassLoader().loadClass(classNames[i]); } - return objectMapper.getTypeFactory().constructParametricType(envelopeClass, classes); + return typeFactory.create(envelopeClass, classes); } public Object deserialize(String payload) { try { if (SerializationUtils.isJson(payload)) { - Msg msg = objectMapper.readValue(payload, Msg.class); - JavaType type = getTargetType(msg); - return objectMapper.readValue(msg.msg, type); + Msg msg = serDes.deserialize(payload, Msg.class); + TypeEnvelop type = getTargetType(msg); + return serDes.deserialize(msg.msg, type); } } catch (Exception e) { log.debug("Deserialization of message {} failed", payload, e); @@ -230,7 +284,7 @@ public T deserialize(byte[] payload, Class clazz) { return null; } try { - return objectMapper.readValue(payload, clazz); + return serDes.deserialize(payload, clazz); } catch (Exception e) { log.debug("Deserialization of message {} failed", new String(payload), e); } @@ -243,10 +297,10 @@ public String serialize(Object payload) { return null; } try { - String msg = objectMapper.writeValueAsString(payload); + byte[] msg = serDes.serialize(payload); Msg message = new Msg(msg, name); - return objectMapper.writeValueAsString(message); - } catch (JacksonException e) { + return serDes.serializeAsString(message); + } catch (Exception e) { log.debug("Serialisation failed", e); return null; } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/JsonMessageConverter.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/JsonMessageConverter.java index 79fad251b..c823d0291 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/JsonMessageConverter.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/JsonMessageConverter.java @@ -18,14 +18,13 @@ import static org.springframework.util.Assert.notNull; -import com.github.sonus21.rqueue.utils.SerializationUtils; +import com.github.sonus21.rqueue.serdes.RqueueSerDes; +import com.github.sonus21.rqueue.serdes.SerializationUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.messaging.Message; import org.springframework.messaging.MessageHeaders; import org.springframework.messaging.converter.MessageConverter; import org.springframework.messaging.support.GenericMessage; -import tools.jackson.core.JacksonException; -import tools.jackson.databind.ObjectMapper; /** * JsonMessageConverter tries to convert to JSON and from JSON to object. @@ -41,15 +40,15 @@ @Slf4j public class JsonMessageConverter implements MessageConverter { - private final ObjectMapper objectMapper; + private final RqueueSerDes serDes; public JsonMessageConverter() { - this.objectMapper = SerializationUtils.createObjectMapper(); + this.serDes = SerializationUtils.getSerDes(); } - public JsonMessageConverter(ObjectMapper objectMapper) { - notNull(objectMapper, "objectMapper cannot be null"); - this.objectMapper = objectMapper; + public JsonMessageConverter(RqueueSerDes serDes) { + notNull(serDes, "serDes cannot be null"); + this.serDes = serDes; } @Override @@ -61,10 +60,10 @@ public Object fromMessage(Message message, Class targetClass) { return null; } if (SerializationUtils.isJson(payload)) { - return objectMapper.readValue(payload, targetClass); + return serDes.deserialize(payload, targetClass); } return null; - } catch (JacksonException | ClassCastException e) { + } catch (Exception e) { log.debug("Deserialization of message {} failed", message, e); return null; } @@ -74,9 +73,8 @@ public Object fromMessage(Message message, Class targetClass) { public Message toMessage(Object payload, MessageHeaders headers) { log.trace("Payload: {} Headers: {}", payload, headers); try { - String msg = objectMapper.writeValueAsString(payload); - return new GenericMessage<>(msg); - } catch (JacksonException e) { + return new GenericMessage<>(serDes.serializeAsString(payload)); + } catch (Exception e) { log.debug("Serialisation failed, Payload: {}", payload, e); return null; } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/MessageConverterProvider.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/MessageConverterProvider.java index 8467618ec..7297a71f1 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/MessageConverterProvider.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/MessageConverterProvider.java @@ -20,7 +20,6 @@ import org.springframework.messaging.Message; import org.springframework.messaging.converter.CompositeMessageConverter; import org.springframework.messaging.converter.JacksonJsonMessageConverter; -import org.springframework.messaging.converter.MappingJackson2MessageConverter; import org.springframework.messaging.converter.MessageConverter; import org.springframework.messaging.converter.SmartMessageConverter; import org.springframework.messaging.converter.StringMessageConverter; @@ -42,7 +41,6 @@ * @see SmartMessageConverter * @see DefaultRqueueMessageConverter * @see JacksonJsonMessageConverter - * @see MappingJackson2MessageConverter */ public interface MessageConverterProvider { diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/RqueueRedisSerializer.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/RqueueRedisSerializer.java index c82b94ce9..245fe1819 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/RqueueRedisSerializer.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/RqueueRedisSerializer.java @@ -17,7 +17,7 @@ package com.github.sonus21.rqueue.converter; import com.fasterxml.jackson.annotation.JsonTypeInfo.As; -import com.github.sonus21.rqueue.utils.SerializationUtils; +import com.github.sonus21.rqueue.serdes.SerializationUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.cache.support.NullValue; import org.springframework.data.redis.serializer.RedisSerializer; @@ -64,11 +64,10 @@ public Object deserialize(byte[] bytes) throws SerializationException { // adapted from spring-data-redis private static class RqueueRedisSerDes implements RedisSerializer { - - private ObjectMapper mapper; + private final ObjectMapper mapper; RqueueRedisSerDes() { - this.mapper = SerializationUtils.createObjectMapper() + this.mapper = SerializationUtils.getObjectMapper() .rebuild() .addModule(new SimpleModule().addSerializer(new NullValueSerializer())) .activateDefaultTyping( diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/EndpointRegistry.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/EndpointRegistry.java index 8c4966596..b8a409665 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/EndpointRegistry.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/EndpointRegistry.java @@ -37,7 +37,15 @@ public final class EndpointRegistry { private static final Object lock = new Object(); - private static final Map queueNameToDetail = new HashMap<>(); + // composite key: "queueName" for single-consumer, "queueName##consumerName" for multi-consumer + private static final Map registry = new HashMap<>(); + // first registered QueueDetail per queue name — used by enqueue / management operations + private static final Map primaryByName = new HashMap<>(); + + private static String compositeKey(QueueDetail qd) { + String cn = qd.getConsumerName(); + return (cn != null && !cn.isEmpty()) ? qd.getName() + "##" + cn : qd.getName(); + } private EndpointRegistry() {} @@ -51,7 +59,7 @@ private EndpointRegistry() {} * @see #get(String, String) */ public static QueueDetail get(String queueName) { - QueueDetail queueDetail = queueNameToDetail.get(queueName); + QueueDetail queueDetail = primaryByName.get(queueName); if (queueDetail == null) { throw new QueueDoesNotExist(queueName); } @@ -68,7 +76,7 @@ public static QueueDetail get(String queueName) { */ public static QueueDetail get(String queueName, String priority) { QueueDetail queueDetail = - queueNameToDetail.get(PriorityUtils.getQueueNameForPriority(queueName, priority)); + primaryByName.get(PriorityUtils.getQueueNameForPriority(queueName, priority)); if (queueDetail == null) { throw new QueueDoesNotExist(queueName); } @@ -77,24 +85,27 @@ public static QueueDetail get(String queueName, String priority) { public static void register(QueueDetail queueDetail) { synchronized (lock) { - if (queueNameToDetail.containsKey(queueDetail.getName())) { + String ck = compositeKey(queueDetail); + if (registry.containsKey(ck)) { throw new OverrideException(queueDetail.getName()); } - queueNameToDetail.put(queueDetail.getName(), queueDetail); + registry.put(ck, queueDetail); + primaryByName.putIfAbsent(queueDetail.getName(), queueDetail); lock.notifyAll(); } } public static void delete() { synchronized (lock) { - queueNameToDetail.clear(); + registry.clear(); + primaryByName.clear(); lock.notifyAll(); } } public static List getActiveQueues() { synchronized (lock) { - List queues = queueNameToDetail.values().stream() + List queues = primaryByName.values().stream() .filter(QueueDetail::isActive) .map(QueueDetail::getName) .collect(Collectors.toList()); @@ -105,9 +116,8 @@ public static List getActiveQueues() { public static List getActiveQueueDetails() { synchronized (lock) { - List queueDetails = queueNameToDetail.values().stream() - .filter(QueueDetail::isActive) - .collect(Collectors.toList()); + List queueDetails = + registry.values().stream().filter(QueueDetail::isActive).collect(Collectors.toList()); lock.notifyAll(); return queueDetails; } @@ -115,7 +125,7 @@ public static List getActiveQueueDetails() { public static Map getActiveQueueMap() { synchronized (lock) { - Map queueDetails = queueNameToDetail.values().stream() + Map queueDetails = primaryByName.values().stream() .filter(QueueDetail::isActive) .collect(Collectors.toMap(QueueDetail::getName, Function.identity())); lock.notifyAll(); @@ -126,7 +136,7 @@ public static Map getActiveQueueMap() { public static String toStr() { StringBuilder builder = new StringBuilder(); synchronized (lock) { - List queueDetails = new ArrayList<>(queueNameToDetail.values()); + List queueDetails = new ArrayList<>(registry.values()); queueDetails.sort(Comparator.comparing(QueueDetail::getName)); for (QueueDetail q : queueDetails) { builder.append(q.toString()); @@ -142,6 +152,6 @@ public static int getActiveQueueCount() { } public static int getRegisteredQueueCount() { - return queueNameToDetail.size(); + return registry.size(); } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueBeanProvider.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueBeanProvider.java index cf95221ea..a9675bcef 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueBeanProvider.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueBeanProvider.java @@ -19,12 +19,13 @@ import com.github.sonus21.rqueue.common.RqueueLockManager; import com.github.sonus21.rqueue.config.RqueueConfig; import com.github.sonus21.rqueue.config.RqueueWebConfig; +import com.github.sonus21.rqueue.core.spi.MessageBroker; import com.github.sonus21.rqueue.core.support.MessageProcessor; import com.github.sonus21.rqueue.dao.RqueueJobDao; import com.github.sonus21.rqueue.dao.RqueueSystemConfigDao; import com.github.sonus21.rqueue.listener.RqueueMessageHandler; import com.github.sonus21.rqueue.metrics.RqueueMetricsCounter; -import com.github.sonus21.rqueue.web.service.RqueueMessageMetadataService; +import com.github.sonus21.rqueue.service.RqueueMessageMetadataService; import com.github.sonus21.rqueue.worker.RqueueWorkerRegistry; import lombok.Getter; import lombok.Setter; @@ -69,4 +70,7 @@ public class RqueueBeanProvider { @Autowired private RqueueConfig rqueueConfig; + + /** Set by the container during initialization; never null after {@code afterPropertiesSet}. */ + private MessageBroker messageBroker; } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueEndpointManager.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueEndpointManager.java index c08627a6f..6274604e9 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueEndpointManager.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueEndpointManager.java @@ -16,6 +16,7 @@ package com.github.sonus21.rqueue.core; +import com.github.sonus21.rqueue.enums.QueueType; import com.github.sonus21.rqueue.listener.QueueDetail; import com.github.sonus21.rqueue.utils.PriorityUtils; import java.util.List; @@ -35,7 +36,18 @@ public interface RqueueEndpointManager { * @param name name of the queue * @param priorities list of priorities to be used while sending message on this queue. */ - void registerQueue(String name, String... priorities); + default void registerQueue(String name, String... priorities) { + registerQueue(name, QueueType.QUEUE, priorities); + } + + /** + * Use this method to register any queue, that's only used for sending message. + * + * @param name name of the queue + * @param type queue type + * @param priorities list of priorities to be used while sending message on this queue. + */ + void registerQueue(String name, QueueType type, String... priorities); /** * Check if a queue is registered. diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueInternalPubSubChannel.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueInternalPubSubChannel.java index 55539d785..ac2517c97 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueInternalPubSubChannel.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueInternalPubSubChannel.java @@ -18,14 +18,14 @@ import com.github.sonus21.rqueue.common.RqueueRedisTemplate; import com.github.sonus21.rqueue.config.RqueueConfig; -import com.github.sonus21.rqueue.converter.GenericMessageConverter.SmartMessageSerDes; import com.github.sonus21.rqueue.converter.RqueueRedisSerializer; import com.github.sonus21.rqueue.listener.RqueueMessageListenerContainer; import com.github.sonus21.rqueue.models.enums.PubSubType; import com.github.sonus21.rqueue.models.event.RqueuePubSubEvent; import com.github.sonus21.rqueue.models.request.PauseUnpauseQueueRequest; +import com.github.sonus21.rqueue.serdes.RqueueSerDes; +import com.github.sonus21.rqueue.serdes.SerializationUtils; import com.github.sonus21.rqueue.utils.Constants; -import com.github.sonus21.rqueue.utils.SerializationUtils; import com.github.sonus21.rqueue.utils.StringUtils; import java.time.Duration; import java.util.UUID; @@ -44,7 +44,7 @@ public class RqueueInternalPubSubChannel implements InitializingBean { private final RqueueRedisTemplate stringRqueueRedisTemplate; private final RqueueRedisSerializer rqueueRedisSerializer; private final RqueueBeanProvider rqueueBeanProvider; - private SmartMessageSerDes smartMessageSerDes; + private final RqueueSerDes serDes = SerializationUtils.getSerDes(); public RqueueInternalPubSubChannel( RqueueRedisListenerContainerFactory rqueueRedisListenerContainerFactory, @@ -63,10 +63,8 @@ public RqueueInternalPubSubChannel( @Override public void afterPropertiesSet() throws Exception { String channel = rqueueConfig.getInternalCommChannelName(); - InternalMessageListener messageListener = new InternalMessageListener(); rqueueRedisListenerContainerFactory.addMessageListener( - messageListener, new ChannelTopic(channel)); - this.smartMessageSerDes = new SmartMessageSerDes(SerializationUtils.createObjectMapper()); + new InternalMessageListener(), new ChannelTopic(channel)); } public void emitPauseUnpauseQueueEvent(PauseUnpauseQueueRequest pauseUnpauseQueueRequest) { @@ -103,24 +101,32 @@ public void onMessage(Message message, byte[] pattern) { private void processEvent(byte[] body) { log.debug("Message on internal channel {}", new String(body)); - RqueuePubSubEvent rqueuePubSubEvent = - smartMessageSerDes.deserialize(body, RqueuePubSubEvent.class); + RqueuePubSubEvent rqueuePubSubEvent; + try { + rqueuePubSubEvent = serDes.deserialize(body, RqueuePubSubEvent.class); + } catch (Exception e) { + log.error("Invalid message on pub-sub channel {}", new String(body), e); + return; + } if (rqueuePubSubEvent == null) { log.error("Invalid message on pub-sub channel {}", new String(body)); return; } - switch (rqueuePubSubEvent.getType()) { - case PAUSE_QUEUE: - PauseUnpauseQueueRequest request = - rqueuePubSubEvent.messageAs(smartMessageSerDes, PauseUnpauseQueueRequest.class); - handlePauseEvent(request); - break; - case QUEUE_CRUD: - String queue = rqueuePubSubEvent.messageAs(smartMessageSerDes, String.class); - rqueueBeanProvider.getRqueueSystemConfigDao().clearCacheByName(queue); - break; - default: - log.error("Unknown event type {}", rqueuePubSubEvent); + try { + switch (rqueuePubSubEvent.getType()) { + case PAUSE_QUEUE: + handlePauseEvent(rqueuePubSubEvent.messageAs(serDes, PauseUnpauseQueueRequest.class)); + break; + case QUEUE_CRUD: + rqueueBeanProvider + .getRqueueSystemConfigDao() + .clearCacheByName(rqueuePubSubEvent.messageAs(serDes, String.class)); + break; + default: + log.error("Unknown event type {}", rqueuePubSubEvent); + } + } catch (Exception e) { + log.error("Failed to process pub-sub event {}", rqueuePubSubEvent, e); } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessage.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessage.java index 10c958b97..546454948 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessage.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessage.java @@ -28,7 +28,24 @@ import org.springframework.messaging.MessageHeaders; /** - * Internal message for Rqueue + * Envelope for a message being processed through Rqueue. Each message maintains its own state + * including serialized content, retry metadata, failure counts, and timing information. + * + *

Message Content. The {@code message} field holds the serialized (JSON) representation + * of the user's object. Use {@link com.github.sonus21.rqueue.core.support.RqueueMessageUtils} + * to convert between serialized and object form. + * + *

Timing and Scheduling. {@code queuedTime} records when the message was enqueued + * (nanosecond precision). {@code processAt} defines the intended processing time (for delayed + * messages). {@code reEnqueuedAt} tracks the last re-queue timestamp following failure. + * + *

Failure Tracking. {@code failureCount} and {@code sourceQueueFailureCount} track + * retries in the current and source queue respectively. Once retries are exhausted, the message + * may be routed to the configured dead-letter queue ({@code sourceQueueName}). + * + *

Periodic Tasks. For messages representing recurring work, the {@code period} field + * (in milliseconds) defines the recurrence interval. Use {@code nextProcessAt()} to compute the + * next scheduling time. */ @Getter @Setter diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/BaseMessageSender.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/BaseMessageSender.java index acc4b5d0d..d3c8d9493 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/BaseMessageSender.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/BaseMessageSender.java @@ -29,13 +29,13 @@ import com.github.sonus21.rqueue.core.RqueueMessageIdGenerator; import com.github.sonus21.rqueue.core.RqueueMessageTemplate; import com.github.sonus21.rqueue.core.impl.MessageSweeper.MessageDeleteRequest; -import com.github.sonus21.rqueue.dao.RqueueStringDao; +import com.github.sonus21.rqueue.enums.QueueType; import com.github.sonus21.rqueue.exception.DuplicateMessageException; import com.github.sonus21.rqueue.listener.QueueDetail; import com.github.sonus21.rqueue.models.db.MessageMetadata; import com.github.sonus21.rqueue.models.enums.MessageStatus; +import com.github.sonus21.rqueue.service.RqueueMessageMetadataService; import com.github.sonus21.rqueue.utils.PriorityUtils; -import com.github.sonus21.rqueue.web.service.RqueueMessageMetadataService; import java.time.Duration; import java.util.Collections; import java.util.HashMap; @@ -53,9 +53,7 @@ abstract class BaseMessageSender { protected final MessageConverter messageConverter; protected final RqueueMessageTemplate messageTemplate; protected final RqueueMessageIdGenerator messageIdGenerator; - - @Autowired - protected RqueueStringDao rqueueStringDao; + protected final com.github.sonus21.rqueue.core.spi.MessageBroker messageBroker; @Autowired protected RqueueConfig rqueueConfig; @@ -65,13 +63,16 @@ abstract class BaseMessageSender { BaseMessageSender( RqueueMessageTemplate messageTemplate, + com.github.sonus21.rqueue.core.spi.MessageBroker messageBroker, MessageConverter messageConverter, MessageHeaders messageHeaders, RqueueMessageIdGenerator messageIdGenerator) { notNull(messageTemplate, "messageTemplate cannot be null"); + notNull(messageBroker, "messageBroker cannot be null"); notNull(messageConverter, "messageConverter cannot be null"); notNull(messageIdGenerator, "messageIdGenerator cannot be null"); this.messageTemplate = messageTemplate; + this.messageBroker = messageBroker; this.messageConverter = messageConverter; this.messageHeaders = messageHeaders; this.messageIdGenerator = messageIdGenerator; @@ -79,6 +80,10 @@ abstract class BaseMessageSender { protected Object storeMessageMetadata( RqueueMessage rqueueMessage, Long delayInMillis, boolean reactive, boolean isUnique) { + boolean skipMetadata = !messageBroker.capabilities().usesPrimaryHandlerDispatch(); + if (skipMetadata) { + return reactive ? reactor.core.publisher.Mono.just(true) : null; + } MessageMetadata messageMetadata = new MessageMetadata(rqueueMessage, MessageStatus.ENQUEUED); Duration duration = rqueueConfig.getMessageDurability(delayInMillis); if (reactive) { @@ -94,24 +99,33 @@ protected Object enqueue( RqueueMessage rqueueMessage, Long delayInMilliSecs, boolean reactive) { + return enqueue(queueDetail, null, rqueueMessage, delayInMilliSecs, reactive); + } + + /** + * Priority-aware enqueue. Always routes through {@link + * com.github.sonus21.rqueue.core.spi.MessageBroker} — the Redis-vs-NATS dispatch lives inside + * each broker implementation. Backends that key off the queue name (Redis) ignore {@code + * priority}; backends that publish to a per-priority destination (NATS) use it to pick the + * subject. Reactive enqueues route through {@code enqueueReactive} so backends with native + * async APIs do not block a thread. + */ + protected Object enqueue( + QueueDetail queueDetail, + String priority, + RqueueMessage rqueueMessage, + Long delayInMilliSecs, + boolean reactive) { if (delayInMilliSecs == null || delayInMilliSecs <= MIN_DELAY) { if (reactive) { - return messageTemplate.addReactiveMessage(queueDetail.getQueueName(), rqueueMessage); - } else { - messageTemplate.addMessage(queueDetail.getQueueName(), rqueueMessage); + return messageBroker.enqueueReactive(queueDetail, rqueueMessage); } + messageBroker.enqueue(queueDetail, priority, rqueueMessage); } else { if (reactive) { - return messageTemplate.addReactiveMessageWithDelay( - queueDetail.getScheduledQueueName(), - queueDetail.getScheduledQueueChannelName(), - rqueueMessage); - } else { - messageTemplate.addMessageWithDelay( - queueDetail.getScheduledQueueName(), - queueDetail.getScheduledQueueChannelName(), - rqueueMessage); + return messageBroker.enqueueWithDelayReactive(queueDetail, rqueueMessage, delayInMilliSecs); } + messageBroker.enqueueWithDelay(queueDetail, rqueueMessage, delayInMilliSecs); } return null; } @@ -123,6 +137,17 @@ protected String pushMessage( Integer retryCount, Long delayInMilliSecs, boolean isUnique) { + return pushMessage(queueName, null, messageId, message, retryCount, delayInMilliSecs, isUnique); + } + + protected String pushMessage( + String queueName, + String priority, + String messageId, + Object message, + Integer retryCount, + Long delayInMilliSecs, + boolean isUnique) { QueueDetail queueDetail = EndpointRegistry.get(queueName); RqueueMessage rqueueMessage = buildMessage( messageIdGenerator, @@ -135,7 +160,7 @@ protected String pushMessage( messageHeaders); try { storeMessageMetadata(rqueueMessage, delayInMilliSecs, false, isUnique); - enqueue(queueDetail, rqueueMessage, delayInMilliSecs, false); + enqueue(queueDetail, priority, rqueueMessage, delayInMilliSecs, false); } catch (DuplicateMessageException e) { log.warn( "Duplicate message enqueue attempted queue: {}, messageId: {}", @@ -177,8 +202,9 @@ protected Object deleteAllMessages(QueueDetail queueDetail) { MessageDeleteRequest.builder().queueDetail(queueDetail).build()); } - protected void registerQueueInternal(String queueName, String... priorities) { + protected void registerQueueInternal(String queueName, QueueType type, String... priorities) { validateQueue(queueName); + messageBroker.validateQueueName(queueName); notNull(priorities, "priorities cannot be null"); Map priorityMap = new HashMap<>(); priorityMap.put(DEFAULT_PRIORITY_KEY, 1); @@ -195,8 +221,10 @@ protected void registerQueueInternal(String queueName, String... priorities) { .processingQueueName(rqueueConfig.getProcessingQueueName(queueName)) .processingQueueChannelName(rqueueConfig.getProcessingQueueChannelName(queueName)) .priority(priorityMap) + .type(type) .build(); EndpointRegistry.register(queueDetail); + notifyBrokerQueueRegistered(queueDetail); for (String priority : priorities) { String suffix = PriorityUtils.getSuffix(priority); queueDetail = QueueDetail.builder() @@ -211,6 +239,11 @@ protected void registerQueueInternal(String queueName, String... priorities) { .priority(Collections.singletonMap(DEFAULT_PRIORITY_KEY, 1)) .build(); EndpointRegistry.register(queueDetail); + notifyBrokerQueueRegistered(queueDetail); } } + + private void notifyBrokerQueueRegistered(QueueDetail queueDetail) { + messageBroker.onQueueRegistered(queueDetail); + } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/MessageSweeper.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/MessageSweeper.java index 16e56459b..b0de6015b 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/MessageSweeper.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/MessageSweeper.java @@ -22,9 +22,9 @@ import com.github.sonus21.rqueue.core.support.RqueueMessageUtils; import com.github.sonus21.rqueue.exception.UnknownSwitchCase; import com.github.sonus21.rqueue.listener.QueueDetail; +import com.github.sonus21.rqueue.service.RqueueMessageMetadataService; import com.github.sonus21.rqueue.utils.RetryableRunnable; import com.github.sonus21.rqueue.utils.StringUtils; -import com.github.sonus21.rqueue.web.service.RqueueMessageMetadataService; import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedList; diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/ReactiveRqueueMessageEnqueuerImpl.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/ReactiveRqueueMessageEnqueuerImpl.java index 6c138b8a5..bb5e7d04f 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/ReactiveRqueueMessageEnqueuerImpl.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/ReactiveRqueueMessageEnqueuerImpl.java @@ -23,6 +23,7 @@ import com.github.sonus21.rqueue.core.RqueueMessage; import com.github.sonus21.rqueue.core.RqueueMessageIdGenerator; import com.github.sonus21.rqueue.core.RqueueMessageTemplate; +import com.github.sonus21.rqueue.core.spi.MessageBroker; import com.github.sonus21.rqueue.core.support.RqueueMessageUtils; import com.github.sonus21.rqueue.exception.DuplicateMessageException; import com.github.sonus21.rqueue.listener.QueueDetail; @@ -31,7 +32,6 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.messaging.MessageHeaders; import org.springframework.messaging.converter.MessageConverter; -import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @Slf4j @@ -40,17 +40,24 @@ public class ReactiveRqueueMessageEnqueuerImpl extends BaseMessageSender public ReactiveRqueueMessageEnqueuerImpl( RqueueMessageTemplate messageTemplate, + MessageBroker messageBroker, MessageConverter messageConverter, MessageHeaders messageHeaders) { - this(messageTemplate, messageConverter, messageHeaders, new UuidV4RqueueMessageIdGenerator()); + this( + messageTemplate, + messageBroker, + messageConverter, + messageHeaders, + new UuidV4RqueueMessageIdGenerator()); } public ReactiveRqueueMessageEnqueuerImpl( RqueueMessageTemplate messageTemplate, + MessageBroker messageBroker, MessageConverter messageConverter, MessageHeaders messageHeaders, RqueueMessageIdGenerator messageIdGenerator) { - super(messageTemplate, messageConverter, messageHeaders, messageIdGenerator); + super(messageTemplate, messageBroker, messageConverter, messageHeaders, messageIdGenerator); } @SuppressWarnings("unchecked") @@ -77,21 +84,18 @@ private Mono pushReactiveMessage( Mono storeResult = (Mono) storeMessageMetadata(rqueueMessage, delayInMilliSecs, true, isUnique); return storeResult.flatMap(success -> { - if (Boolean.TRUE.equals(success)) { - Object result = enqueue(queueDetail, rqueueMessage, delayInMilliSecs, true); - Mono enqueueMono; - if (result instanceof Flux) { - enqueueMono = ((Flux) result).next(); - } else if (result instanceof Mono) { - enqueueMono = (Mono) result; - } else { - return Mono.error( - new IllegalStateException("Unexpected enqueue result type: " + result.getClass())); - } - return enqueueMono.flatMap(ignore -> monoConverter.apply(rqueueMessage)); - } else { + if (!Boolean.TRUE.equals(success)) { return Mono.error(new DuplicateMessageException(rqueueMessage.getId())); } + Mono brokerMono; + if (delayInMilliSecs == null + || delayInMilliSecs <= com.github.sonus21.rqueue.utils.Constants.MIN_DELAY) { + brokerMono = messageBroker.enqueueReactive(queueDetail, rqueueMessage); + } else { + brokerMono = + messageBroker.enqueueWithDelayReactive(queueDetail, rqueueMessage, delayInMilliSecs); + } + return brokerMono.then(Mono.defer(() -> monoConverter.apply(rqueueMessage))); }); } catch (Exception e) { log.error( diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueEndpointManagerImpl.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueEndpointManagerImpl.java index 907dbe8fa..149b91fce 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueEndpointManagerImpl.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueEndpointManagerImpl.java @@ -21,15 +21,16 @@ import com.github.sonus21.rqueue.core.RqueueMessageIdGenerator; import com.github.sonus21.rqueue.core.RqueueMessageTemplate; import com.github.sonus21.rqueue.dao.RqueueSystemConfigDao; +import com.github.sonus21.rqueue.enums.QueueType; import com.github.sonus21.rqueue.exception.QueueDoesNotExist; import com.github.sonus21.rqueue.listener.QueueDetail; import com.github.sonus21.rqueue.models.db.QueueConfig; import com.github.sonus21.rqueue.models.request.PauseUnpauseQueueRequest; import com.github.sonus21.rqueue.models.response.BaseResponse; +import com.github.sonus21.rqueue.service.RqueueUtilityService; import com.github.sonus21.rqueue.utils.Constants; import com.github.sonus21.rqueue.utils.PriorityUtils; import com.github.sonus21.rqueue.utils.Validator; -import com.github.sonus21.rqueue.web.service.RqueueUtilityService; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -50,22 +51,29 @@ public class RqueueEndpointManagerImpl extends BaseMessageSender implements Rque public RqueueEndpointManagerImpl( RqueueMessageTemplate messageTemplate, + com.github.sonus21.rqueue.core.spi.MessageBroker messageBroker, MessageConverter messageConverter, MessageHeaders messageHeaders) { - this(messageTemplate, messageConverter, messageHeaders, new UuidV4RqueueMessageIdGenerator()); + this( + messageTemplate, + messageBroker, + messageConverter, + messageHeaders, + new UuidV4RqueueMessageIdGenerator()); } public RqueueEndpointManagerImpl( RqueueMessageTemplate messageTemplate, + com.github.sonus21.rqueue.core.spi.MessageBroker messageBroker, MessageConverter messageConverter, MessageHeaders messageHeaders, RqueueMessageIdGenerator messageIdGenerator) { - super(messageTemplate, messageConverter, messageHeaders, messageIdGenerator); + super(messageTemplate, messageBroker, messageConverter, messageHeaders, messageIdGenerator); } @Override - public void registerQueue(String name, String... priorities) { - registerQueueInternal(name, priorities); + public void registerQueue(String name, QueueType type, String... priorities) { + registerQueueInternal(name, type, priorities); } @Override diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageEnqueuerImpl.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageEnqueuerImpl.java index 6e17fa5e7..c2fe8e4cc 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageEnqueuerImpl.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageEnqueuerImpl.java @@ -27,6 +27,7 @@ import com.github.sonus21.rqueue.core.RqueueMessageEnqueuer; import com.github.sonus21.rqueue.core.RqueueMessageIdGenerator; import com.github.sonus21.rqueue.core.RqueueMessageTemplate; +import com.github.sonus21.rqueue.core.spi.MessageBroker; import com.github.sonus21.rqueue.utils.PriorityUtils; import java.util.Objects; import lombok.extern.slf4j.Slf4j; @@ -38,17 +39,24 @@ public class RqueueMessageEnqueuerImpl extends BaseMessageSender implements Rque public RqueueMessageEnqueuerImpl( RqueueMessageTemplate messageTemplate, + MessageBroker messageBroker, MessageConverter messageConverter, MessageHeaders messageHeaders) { - this(messageTemplate, messageConverter, messageHeaders, new UuidV4RqueueMessageIdGenerator()); + this( + messageTemplate, + messageBroker, + messageConverter, + messageHeaders, + new UuidV4RqueueMessageIdGenerator()); } public RqueueMessageEnqueuerImpl( RqueueMessageTemplate messageTemplate, + MessageBroker messageBroker, MessageConverter messageConverter, MessageHeaders messageHeaders, RqueueMessageIdGenerator messageIdGenerator) { - super(messageTemplate, messageConverter, messageHeaders, messageIdGenerator); + super(messageTemplate, messageBroker, messageConverter, messageHeaders, messageIdGenerator); } private void validateBasic(String queue, Object message) { @@ -99,13 +107,7 @@ public boolean enqueueWithRetry( public String enqueueWithPriority(String queueName, String priority, Object message) { validateBasic(queueName, message); validatePriority(priority); - return pushMessage( - PriorityUtils.getQueueNameForPriority(queueName, priority), - null, - message, - null, - null, - false); + return pushMessageForPriority(queueName, priority, null, message, null); } @Override @@ -113,14 +115,35 @@ public boolean enqueueWithPriority( String queueName, String priority, String messageId, Object message) { validateWithId(queueName, messageId, message); validatePriority(priority); + return pushMessageForPriority(queueName, priority, messageId, message, null) != null; + } + + /** + * Routes priority-aware enqueues: + * + *

    + *
  • Redis-style backends (capabilities advertise {@code usesPrimaryHandlerDispatch}): uses + * the suffixed queue name ({@code PriorityUtils.getQueueNameForPriority}). Priority is + * encoded in the queue name; the broker ignores the {@code priority} param. + *
  • Backends with per-priority routing (e.g. NATS): uses the base queue name and passes the + * priority through to + * {@link com.github.sonus21.rqueue.core.spi.MessageBroker#enqueue(QueueDetail, String, + * RqueueMessage)} so the broker picks the per-priority destination (subject/stream). + *
+ */ + private String pushMessageForPriority( + String queueName, String priority, String messageId, Object message, Long delayMs) { + if (!messageBroker.capabilities().usesPrimaryHandlerDispatch()) { + return pushMessage(queueName, priority, messageId, message, null, delayMs, false); + } return pushMessage( - PriorityUtils.getQueueNameForPriority(queueName, priority), - messageId, - message, - null, - null, - false) - != null; + PriorityUtils.getQueueNameForPriority(queueName, priority), + priority, + messageId, + message, + null, + delayMs, + false); } @Override diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageManagerImpl.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageManagerImpl.java index 932f363c5..bccd8a309 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageManagerImpl.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageManagerImpl.java @@ -19,7 +19,6 @@ import static org.springframework.util.Assert.isTrue; import static org.springframework.util.Assert.notNull; -import com.github.sonus21.rqueue.common.RqueueLockManager; import com.github.sonus21.rqueue.core.EndpointRegistry; import com.github.sonus21.rqueue.core.RqueueMessage; import com.github.sonus21.rqueue.core.RqueueMessageIdGenerator; @@ -35,7 +34,6 @@ import java.util.ArrayList; import java.util.List; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.messaging.Message; import org.springframework.messaging.MessageHeaders; import org.springframework.messaging.converter.MessageConverter; @@ -43,23 +41,26 @@ @Slf4j public class RqueueMessageManagerImpl extends BaseMessageSender implements RqueueMessageManager { - - @Autowired - private RqueueLockManager rqueueLockManager; - public RqueueMessageManagerImpl( RqueueMessageTemplate messageTemplate, + com.github.sonus21.rqueue.core.spi.MessageBroker messageBroker, MessageConverter messageConverter, MessageHeaders messageHeaders) { - this(messageTemplate, messageConverter, messageHeaders, new UuidV4RqueueMessageIdGenerator()); + this( + messageTemplate, + messageBroker, + messageConverter, + messageHeaders, + new UuidV4RqueueMessageIdGenerator()); } public RqueueMessageManagerImpl( RqueueMessageTemplate messageTemplate, + com.github.sonus21.rqueue.core.spi.MessageBroker messageBroker, MessageConverter messageConverter, MessageHeaders messageHeaders, RqueueMessageIdGenerator messageIdGenerator) { - super(messageTemplate, messageConverter, messageHeaders, messageIdGenerator); + super(messageTemplate, messageBroker, messageConverter, messageHeaders, messageIdGenerator); } @Override diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageTemplateImpl.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageTemplateImpl.java index 33971cd66..da6923d31 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageTemplateImpl.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageTemplateImpl.java @@ -62,7 +62,9 @@ public RqueueMessageTemplateImpl( RedisConnectionFactory redisConnectionFactory, ReactiveRedisConnectionFactory reactiveRedisConnectionFactory) { super(redisConnectionFactory); - this.scriptExecutor = new DefaultScriptExecutor<>(redisTemplate); + // On the NATS backend path the connection factory is null and the Redis-script executors + // are never invoked; leaving them null fails fast if anyone does try to use them. + this.scriptExecutor = redisTemplate == null ? null : new DefaultScriptExecutor<>(redisTemplate); if (reactiveRedisConnectionFactory != null) { this.reactiveRedisTemplate = new ReactiveRqueueRedisTemplate<>(reactiveRedisConnectionFactory); diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/middleware/HandlerMiddleware.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/middleware/HandlerMiddleware.java index 027053fe1..47dccb41f 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/middleware/HandlerMiddleware.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/middleware/HandlerMiddleware.java @@ -38,10 +38,17 @@ public HandlerMiddleware(RqueueMessageHandler rqueueMessageHandler) { public void handle(Job job, Callable next) throws Exception { Execution execution = job.getLatestExecution(); RqueueMessage rqueueMessage = job.getRqueueMessage(); + // Use the pre-decoded user message when available so that Spring's + // PayloadMethodArgumentResolver does not short-circuit on type-assignability + // and return the raw JSON-encoded envelope (e.g. {"msg":"...","name":"..."}) + // instead of the actual user object. + Object userMessage = job.getMessage(); + Object payload = (userMessage != null) ? userMessage : rqueueMessage.getMessage(); Message message = MessageBuilder.createMessage( - rqueueMessage.getMessage(), + payload, buildMessageHeaders( job.getQueueDetail().getName(), + job.getQueueDetail().getConsumerName(), rqueueMessage, job, execution, diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/spi/Capabilities.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/spi/Capabilities.java new file mode 100644 index 000000000..b2b26e1bb --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/spi/Capabilities.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2020-2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ + +package com.github.sonus21.rqueue.core.spi; + +public record Capabilities( + boolean supportsDelayedEnqueue, + boolean supportsScheduledIntrospection, + boolean supportsCronJobs, + boolean usesPrimaryHandlerDispatch) { + public static final Capabilities REDIS_DEFAULTS = new Capabilities(true, true, true, true); +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/spi/MessageBroker.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/spi/MessageBroker.java new file mode 100644 index 000000000..8a72e3454 --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/spi/MessageBroker.java @@ -0,0 +1,206 @@ +/* + * Copyright (c) 2020-2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ + +package com.github.sonus21.rqueue.core.spi; + +import com.github.sonus21.rqueue.core.RqueueMessage; +import com.github.sonus21.rqueue.listener.QueueDetail; +import java.time.Duration; +import java.util.List; +import java.util.function.Consumer; +import reactor.core.publisher.Mono; + +/** + * Internal SPI. Subject to change. Application code must not depend on this directly. + */ +public interface MessageBroker { + void enqueue(QueueDetail q, RqueueMessage m); + + /** + * Priority-aware enqueue overload. Implementations that route to a per-priority destination + * (e.g. a NATS subject suffixed with the priority name) override this. The default delegates + * to {@link #enqueue(QueueDetail, RqueueMessage)} so backends without per-priority routing + * (Redis already encodes priority in the queue name) keep their existing behavior. + * + * @param q queue detail (already priority-suffixed for backends that key off queue name) + * @param priority priority name as declared on {@code @RqueueListener.priority}; may be + * {@code null} or empty for the default priority bucket + * @param m message to publish + */ + default void enqueue(QueueDetail q, String priority, RqueueMessage m) { + enqueue(q, m); + } + + void enqueueWithDelay(QueueDetail q, RqueueMessage m, long delayMs); + + /** + * Called by {@code RqueueEndpointManager.registerQueue} after a queue is added to the registry. + * Backends that need to provision resources (e.g. JetStream streams) at registration time should + * override this. The default is a no-op so Redis and other backends are unaffected. + */ + default void onQueueRegistered(QueueDetail q) {} + + /** + * Validate the queue name against backend-specific rules. Called from every queue-registration + * path ({@code RqueueEndpointManager.registerQueue} and the {@code @RqueueListener} bootstrap) + * before the queue is added to the registry, so an illegal name fails fast with a clear error + * instead of surfacing later as an opaque NATS / driver-side rejection. + * + *

Default is a no-op — backends like Redis accept any non-empty name. + * + * @throws IllegalArgumentException if {@code queueName} is not legal for this backend + */ + default void validateQueueName(String queueName) {} + + /** + * Reactive variant of {@link #enqueue(QueueDetail, RqueueMessage)}. The default falls back to the + * blocking implementation wrapped in {@code Mono.fromRunnable}; backends with native async + * publish APIs (e.g. JetStream) should override this to avoid blocking the calling thread. + */ + default Mono enqueueReactive(QueueDetail q, RqueueMessage m) { + return Mono.fromRunnable(() -> enqueue(q, m)); + } + + /** + * Reactive variant of {@link #enqueueWithDelay(QueueDetail, RqueueMessage, long)}. The default + * falls back to the blocking implementation. Backends that do not support delayed enqueue should + * override this to return {@code Mono.error(new UnsupportedOperationException(...))}. + */ + default Mono enqueueWithDelayReactive(QueueDetail q, RqueueMessage m, long delayMs) { + return Mono.fromRunnable(() -> enqueueWithDelay(q, m, delayMs)); + } + + List pop(QueueDetail q, String consumerName, int batch, Duration wait); + + /** + * Priority-aware pop overload. Implementations that route to a per-priority stream/consumer + * override this; the default delegates to + * {@link #pop(QueueDetail, String, int, Duration)}. + * + * @param q queue detail + * @param priority priority name; {@code null} or empty for the default bucket + * @param consumerName durable consumer name (already priority-suffixed by the caller for + * backends that key off the consumer name) + * @param batch maximum messages to fetch + * @param wait fetch wait duration + */ + default List pop( + QueueDetail q, String priority, String consumerName, int batch, Duration wait) { + return pop(q, consumerName, batch, wait); + } + + boolean ack(QueueDetail q, RqueueMessage m); + + boolean nack(QueueDetail q, RqueueMessage m, long retryDelayMs); + + long moveExpired(QueueDetail q, long now, int batch); + + List peek(QueueDetail q, long offset, long count); + + /** + * Remove {@code old} from the processing store and re-enqueue {@code updated} for retry. + * {@code delayMs <= 0} means immediate; {@code delayMs > 0} means schedule after that delay. + * Backends without a processing store (e.g. NATS) default to a plain nack. + */ + default void parkForRetry(QueueDetail q, RqueueMessage old, RqueueMessage updated, long delayMs) { + nack(q, updated, delayMs); + } + + /** + * Remove {@code old} from the processing store and enqueue {@code updated} to {@code targetQueue}. + * {@code delayMs <= 0} means immediate (list push); {@code delayMs > 0} means schedule (sorted-set). + * Backends without a processing store default to a plain enqueue to the DLQ. + */ + default void moveToDlq( + QueueDetail source, + String targetQueue, + RqueueMessage old, + RqueueMessage updated, + long delayMs) { + if (delayMs > 0) { + enqueueWithDelay(source, updated, delayMs); + } else { + enqueue(source, updated); + } + } + + /** + * Schedule the next execution of a periodic message. + * {@code messageKey} is the deduplication key; {@code expirySeconds} is the TTL for that key. + * Backends that don't support server-side scheduling default to a delayed enqueue. + */ + default void scheduleNext( + QueueDetail q, String messageKey, RqueueMessage message, long expirySeconds) { + long delayMs = Math.max(0, message.getProcessAt() - System.currentTimeMillis()); + enqueueWithDelay(q, message, delayMs); + } + + /** + * Returns the score (epoch-ms deadline) of {@code m} in the processing store, or {@code null} + * if the backend does not track per-message visibility (e.g. NATS uses consumer-level AckWait). + */ + default Long getVisibilityTimeoutScore(QueueDetail q, RqueueMessage m) { + return null; + } + + /** + * Adds {@code deltaMs} to the visibility timeout of {@code m} in the processing store. + * Returns {@code false} if the backend does not support per-message lease extension. + */ + default boolean extendVisibilityTimeout(QueueDetail q, RqueueMessage m, long deltaMs) { + return false; + } + + long size(QueueDetail q); + + /** + * Short label for the storage backend shown in the dashboard "Queue Storage Footprint" section + * header (e.g. "Redis", "NATS"). Defaults to "Redis". + */ + default String storageKicker() { + return "Redis"; + } + + /** + * One-line description for the storage backend shown below the footprint section heading. + * Defaults to the Redis description. + */ + default String storageDescription() { + return "Underlying Redis structures for the queues visible on this page."; + } + + /** + * Display name for the primary storage unit backing the given queue's messages (pending, + * in-flight, and completed). Returns {@code null} to fall back to the Redis key name. + */ + default String storageDisplayName(QueueDetail q) { + return null; + } + + /** + * Display name for the dead-letter storage unit of the given queue. Returns {@code null} to + * fall back to the DLQ key name stored in {@code DeadLetterQueue}. + */ + default String dlqStorageDisplayName(QueueDetail q) { + return null; + } + + AutoCloseable subscribe(String channel, Consumer handler); + + void publish(String channel, String payload); + + Capabilities capabilities(); +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/spi/MessageBrokerFactory.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/spi/MessageBrokerFactory.java new file mode 100644 index 000000000..06a7ff3d6 --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/spi/MessageBrokerFactory.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2020-2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ + +package com.github.sonus21.rqueue.core.spi; + +import java.util.Map; + +public interface MessageBrokerFactory { + String name(); + + MessageBroker create(Map config); +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/spi/MessageBrokerLoader.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/spi/MessageBrokerLoader.java new file mode 100644 index 000000000..2b6466fa0 --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/spi/MessageBrokerLoader.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2020-2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ + +package com.github.sonus21.rqueue.core.spi; + +import java.util.Map; +import java.util.ServiceLoader; + +public final class MessageBrokerLoader { + + private MessageBrokerLoader() {} + + public static MessageBroker load(String name, Map config) { + for (MessageBrokerFactory f : ServiceLoader.load(MessageBrokerFactory.class)) { + if (f.name().equalsIgnoreCase(name)) { + return f.create(config); + } + } + throw new IllegalArgumentException("No MessageBrokerFactory found for backend: " + name); + } +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/spi/redis/RedisMessageBroker.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/spi/redis/RedisMessageBroker.java new file mode 100644 index 000000000..2351f820a --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/spi/redis/RedisMessageBroker.java @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2020-2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ + +package com.github.sonus21.rqueue.core.spi.redis; + +import com.github.sonus21.rqueue.core.RqueueMessage; +import com.github.sonus21.rqueue.core.RqueueMessageTemplate; +import com.github.sonus21.rqueue.core.spi.Capabilities; +import com.github.sonus21.rqueue.core.spi.MessageBroker; +import com.github.sonus21.rqueue.listener.QueueDetail; +import com.github.sonus21.rqueue.models.MessageMoveResult; +import com.github.sonus21.rqueue.utils.RedisUtils; +import java.time.Duration; +import java.util.List; +import java.util.function.Consumer; +import org.springframework.data.redis.connection.Message; +import org.springframework.data.redis.connection.MessageListener; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.data.redis.listener.ChannelTopic; +import org.springframework.data.redis.listener.RedisMessageListenerContainer; +import reactor.core.publisher.Mono; + +/** + * Default {@link MessageBroker} implementation that delegates to the existing Redis-backed + * code path via {@link RqueueMessageTemplate}. + * + *

This is a thin Phase 1 wrapper: every method routes to the same call site that the existing + * public API uses. No Lua scripts, DAO impls, or message flows are duplicated here. The intent is + * to introduce the SPI seam without changing observable Redis behavior. + */ +public class RedisMessageBroker implements MessageBroker { + + private final RqueueMessageTemplate template; + private final RedisMessageListenerContainer pubSubContainer; + + public RedisMessageBroker(RqueueMessageTemplate template) { + this(template, null); + } + + public RedisMessageBroker( + RqueueMessageTemplate template, RedisMessageListenerContainer pubSubContainer) { + if (template == null) { + throw new IllegalArgumentException("template cannot be null"); + } + this.template = template; + this.pubSubContainer = pubSubContainer; + } + + public RqueueMessageTemplate getTemplate() { + return template; + } + + @Override + public void enqueue(QueueDetail q, RqueueMessage m) { + template.addMessage(q.getQueueName(), m); + } + + @Override + public void enqueueWithDelay(QueueDetail q, RqueueMessage m, long delayMs) { + // Delegate to existing scheduled-queue add path; processAt is encoded on the message. + template.addMessageWithDelay(q.getScheduledQueueName(), q.getScheduledQueueChannelName(), m); + } + + /** + * Override the SPI default (which wraps the blocking call in {@code Mono.fromRunnable}) so + * reactive callers stay on the reactive Redis driver and never block a thread. + */ + @Override + public Mono enqueueReactive(QueueDetail q, RqueueMessage m) { + return template.addReactiveMessage(q.getQueueName(), m).then(); + } + + /** Reactive scheduled-queue equivalent of {@link #enqueueWithDelay}. */ + @Override + public Mono enqueueWithDelayReactive(QueueDetail q, RqueueMessage m, long delayMs) { + return template + .addReactiveMessageWithDelay(q.getScheduledQueueName(), q.getScheduledQueueChannelName(), m) + .then(); + } + + @Override + public List pop(QueueDetail q, String consumerName, int batch, Duration wait) { + return template.pop( + q.getQueueName(), + q.getProcessingQueueName(), + q.getProcessingQueueChannelName(), + q.getVisibilityTimeout(), + batch); + } + + @Override + public boolean ack(QueueDetail q, RqueueMessage m) { + Long removed = template.removeElementFromZset(q.getProcessingQueueName(), m); + return removed != null && removed > 0; + } + + @Override + public boolean nack(QueueDetail q, RqueueMessage m, long retryDelayMs) { + if (retryDelayMs <= 0) { + template.moveMessage(q.getProcessingQueueName(), q.getQueueName(), m, m); + } else { + template.moveMessageWithDelay( + q.getProcessingQueueName(), q.getScheduledQueueName(), m, m, retryDelayMs); + } + return true; + } + + @Override + public long moveExpired(QueueDetail q, long now, int batch) { + MessageMoveResult result = + template.moveMessageZsetToList(q.getScheduledQueueName(), q.getQueueName(), batch); + return result == null ? 0L : result.getNumberOfMessages(); + } + + @Override + public List peek(QueueDetail q, long offset, long count) { + long end = (count <= 0) ? -1L : offset + count - 1; + return template.readFromList(q.getQueueName(), offset, end); + } + + @Override + public long size(QueueDetail q) { + RedisTemplate rt = template.getTemplate(); + Long size = rt.opsForList().size(q.getQueueName()); + return size == null ? 0L : size; + } + + @Override + public AutoCloseable subscribe(String channel, Consumer handler) { + if (pubSubContainer == null) { + throw new IllegalStateException( + "RedisMessageListenerContainer not configured for RedisMessageBroker; subscribe is" + + " unavailable"); + } + final ChannelTopic topic = new ChannelTopic(channel); + final MessageListener listener = new MessageListener() { + @Override + public void onMessage(Message message, byte[] pattern) { + byte[] body = message.getBody(); + if (body == null) { + return; + } + handler.accept(new String(body)); + } + }; + pubSubContainer.addMessageListener(listener, topic); + return () -> pubSubContainer.removeMessageListener(listener, topic); + } + + @Override + public void publish(String channel, String payload) { + template.getTemplate().convertAndSend(channel, payload); + } + + @Override + public void parkForRetry(QueueDetail q, RqueueMessage old, RqueueMessage updated, long delayMs) { + if (delayMs <= 0) { + template.moveMessage(q.getProcessingQueueName(), q.getQueueName(), old, updated); + } else { + template.moveMessageWithDelay( + q.getProcessingQueueName(), q.getScheduledQueueName(), old, updated, delayMs); + } + } + + @Override + public void moveToDlq( + QueueDetail source, + String targetQueue, + RqueueMessage old, + RqueueMessage updated, + long delayMs) { + RedisUtils.executePipeLine( + template.getTemplate(), (connection, keySerializer, valueSerializer) -> { + byte[] updatedBytes = valueSerializer.serialize(updated); + byte[] oldBytes = valueSerializer.serialize(old); + byte[] processingQueueBytes = keySerializer.serialize(source.getProcessingQueueName()); + byte[] targetQueueBytes = keySerializer.serialize(targetQueue); + if (delayMs > 0) { + connection.zAdd(targetQueueBytes, delayMs, updatedBytes); + } else { + connection.lPush(targetQueueBytes, updatedBytes); + } + connection.zRem(processingQueueBytes, oldBytes); + }); + } + + @Override + public void scheduleNext( + QueueDetail q, String messageKey, RqueueMessage message, long expirySeconds) { + template.scheduleMessage(q.getScheduledQueueName(), messageKey, message, expirySeconds); + } + + @Override + public Long getVisibilityTimeoutScore(QueueDetail q, RqueueMessage m) { + return template.getScore(q.getProcessingQueueName(), m); + } + + @Override + public boolean extendVisibilityTimeout(QueueDetail q, RqueueMessage m, long deltaMs) { + return template.addScore(q.getProcessingQueueName(), m, deltaMs); + } + + @Override + public Capabilities capabilities() { + return Capabilities.REDIS_DEFAULTS; + } +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/enums/QueueType.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/enums/QueueType.java new file mode 100644 index 000000000..f6fa26675 --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/enums/QueueType.java @@ -0,0 +1,12 @@ +package com.github.sonus21.rqueue.enums; + +public enum QueueType { + /** + * WorkQueue semantics: competing consumers, each message delivered to exactly one listener. + */ + QUEUE, + /** + * Stream/fan-out semantics: every independent listener group receives all messages. + */ + STREAM +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/exception/BackendCapabilityException.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/exception/BackendCapabilityException.java new file mode 100644 index 000000000..839e4edaa --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/exception/BackendCapabilityException.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ + +package com.github.sonus21.rqueue.exception; + +/** + * Raised when a web/dashboard operation is invoked against a backend that does not implement the + * primitive it requires (e.g. atomic positional message moves on JetStream). Mapped to HTTP 501 by + * {@code RqueueWebExceptionAdvice}; carries the backend identifier and operation name so callers + * can distinguish "not supported here" from generic failures. + */ +public class BackendCapabilityException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + private final String backend; + private final String operation; + + public BackendCapabilityException(String backend, String operation, String reason) { + super(reason); + this.backend = backend; + this.operation = operation; + } + + public String getBackend() { + return backend; + } + + public String getOperation() { + return operation; + } +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/DefaultRqueuePoller.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/DefaultRqueuePoller.java index 1c054d4e5..0d7ad59e9 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/DefaultRqueuePoller.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/DefaultRqueuePoller.java @@ -38,6 +38,7 @@ class DefaultRqueuePoller extends RqueueMessagePoller { private final QueueThreadPool queueThreadPool; DefaultRqueuePoller( + String pollerKey, QueueDetail queueDetail, QueueThreadPool queueThreadPool, RqueueBeanProvider rqueueBeanProvider, @@ -96,6 +97,8 @@ void poll() { logNotAvailable(); TimeoutUtils.sleepLog(pollingInterval, false); } else { + // Pass the pollerKey (queues[0]) so pollAndExecute's isQueueActive check uses the same + // key that queueRunningState was keyed with (may be "queueName##consumerName"). super.poll(-1, queueDetail.getName(), queueDetail, queueThreadPool); lastNotAvailableAt = null; } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/JobImpl.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/JobImpl.java index 533ee736c..dd8b5a02f 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/JobImpl.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/JobImpl.java @@ -20,10 +20,10 @@ import com.github.sonus21.rqueue.config.RqueueConfig; import com.github.sonus21.rqueue.core.Job; import com.github.sonus21.rqueue.core.RqueueMessage; -import com.github.sonus21.rqueue.core.RqueueMessageTemplate; import com.github.sonus21.rqueue.core.context.Context; import com.github.sonus21.rqueue.core.context.DefaultContext; import com.github.sonus21.rqueue.core.middleware.TimeProviderMiddleware; +import com.github.sonus21.rqueue.core.spi.MessageBroker; import com.github.sonus21.rqueue.dao.RqueueJobDao; import com.github.sonus21.rqueue.models.db.Execution; import com.github.sonus21.rqueue.models.db.MessageMetadata; @@ -31,9 +31,9 @@ import com.github.sonus21.rqueue.models.enums.ExecutionStatus; import com.github.sonus21.rqueue.models.enums.JobStatus; import com.github.sonus21.rqueue.models.enums.MessageStatus; +import com.github.sonus21.rqueue.service.RqueueMessageMetadataService; import com.github.sonus21.rqueue.utils.Constants; import com.github.sonus21.rqueue.utils.TimeoutUtils; -import com.github.sonus21.rqueue.web.service.RqueueMessageMetadataService; import java.io.Serializable; import java.time.Duration; import java.util.List; @@ -50,7 +50,7 @@ public class JobImpl implements Job { public final Duration expiry; private final RqueueJobDao rqueueJobDao; private final RqueueMessageMetadataService messageMetadataService; - private final RqueueMessageTemplate rqueueMessageTemplate; + private final MessageBroker messageBroker; private final RqueueLockManager rqueueLockManager; private final RqueueConfig rqueueConfig; private final QueueDetail queueDetail; @@ -66,7 +66,7 @@ public JobImpl( RqueueConfig rqueueConfig, RqueueMessageMetadataService messageMetadataService, RqueueJobDao rqueueJobDao, - RqueueMessageTemplate rqueueMessageTemplate, + MessageBroker messageBroker, RqueueLockManager rqueueLockManager, QueueDetail queueDetail, MessageMetadata messageMetadata, @@ -76,7 +76,7 @@ public JobImpl( this.rqueueJobDao = rqueueJobDao; this.messageMetadataService = messageMetadataService; this.rqueueConfig = rqueueConfig; - this.rqueueMessageTemplate = rqueueMessageTemplate; + this.messageBroker = messageBroker; this.queueDetail = queueDetail; this.userMessage = userMessage; this.postProcessingHandler = postProcessingHandler; @@ -143,8 +143,7 @@ public void checkIn(Serializable message) { @Override public Duration getVisibilityTimeout() { - Long score = rqueueMessageTemplate.getScore( - queueDetail.getProcessingQueueName(), rqueueJob.getRqueueMessage()); + Long score = messageBroker.getVisibilityTimeoutScore(queueDetail, rqueueJob.getRqueueMessage()); if (score == null || score <= 0) { return Duration.ZERO; } @@ -154,10 +153,8 @@ public Duration getVisibilityTimeout() { @Override public boolean updateVisibilityTimeout(Duration deltaDuration) { - return rqueueMessageTemplate.addScore( - queueDetail.getProcessingQueueName(), - rqueueJob.getRqueueMessage(), - deltaDuration.toMillis()); + return messageBroker.extendVisibilityTimeout( + queueDetail, rqueueJob.getRqueueMessage(), deltaDuration.toMillis()); } @Override diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/MappingInformation.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/MappingInformation.java index 9fec14881..18fb2b1ba 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/MappingInformation.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/MappingInformation.java @@ -19,6 +19,7 @@ import static com.github.sonus21.rqueue.utils.Constants.DELTA_BETWEEN_RE_ENQUEUE_TIME; import static com.github.sonus21.rqueue.utils.Constants.MIN_EXECUTION_TIME; +import com.github.sonus21.rqueue.enums.QueueType; import com.github.sonus21.rqueue.models.Concurrency; import java.util.Map; import java.util.Set; @@ -35,6 +36,9 @@ class MappingInformation implements Comparable { @EqualsAndHashCode.Include private final Set queueNames; + @EqualsAndHashCode.Include + private final String consumerName; + private final int numRetry; private final String deadLetterQueueName; private final boolean deadLetterConsumerEnabled; @@ -47,6 +51,9 @@ class MappingInformation implements Comparable { private final int batchSize; private final Set> doNotRetry; + @Builder.Default + private final QueueType queueType = QueueType.QUEUE; + @Override public String toString() { return String.join(", ", queueNames); diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/PostProcessingHandler.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/PostProcessingHandler.java index e9b18dcae..5090f524b 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/PostProcessingHandler.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/PostProcessingHandler.java @@ -18,7 +18,7 @@ import com.github.sonus21.rqueue.config.RqueueWebConfig; import com.github.sonus21.rqueue.core.RqueueMessage; -import com.github.sonus21.rqueue.core.RqueueMessageTemplate; +import com.github.sonus21.rqueue.core.spi.MessageBroker; import com.github.sonus21.rqueue.dao.RqueueSystemConfigDao; import com.github.sonus21.rqueue.exception.UnknownSwitchCase; import com.github.sonus21.rqueue.models.db.QueueConfig; @@ -26,7 +26,6 @@ import com.github.sonus21.rqueue.models.enums.MessageStatus; import com.github.sonus21.rqueue.models.event.RqueueExecutionEvent; import com.github.sonus21.rqueue.utils.PrefixLogger; -import com.github.sonus21.rqueue.utils.RedisUtils; import com.github.sonus21.rqueue.utils.backoff.FixedTaskExecutionBackOff; import com.github.sonus21.rqueue.utils.backoff.TaskExecutionBackOff; import java.io.Serializable; @@ -40,7 +39,7 @@ class PostProcessingHandler extends PrefixLogger { private final ApplicationEventPublisher applicationEventPublisher; private final RqueueWebConfig rqueueWebConfig; - private final RqueueMessageTemplate rqueueMessageTemplate; + private final MessageBroker broker; private final TaskExecutionBackOff taskExecutionBackoff; private final MessageProcessorHandler messageProcessorHandler; private final RqueueSystemConfigDao rqueueSystemConfigDao; @@ -48,14 +47,14 @@ class PostProcessingHandler extends PrefixLogger { PostProcessingHandler( RqueueWebConfig rqueueWebConfig, ApplicationEventPublisher applicationEventPublisher, - RqueueMessageTemplate rqueueMessageTemplate, + MessageBroker broker, TaskExecutionBackOff taskExecutionBackoff, MessageProcessorHandler messageProcessorHandler, RqueueSystemConfigDao rqueueSystemConfigDao) { super(log, null); this.applicationEventPublisher = applicationEventPublisher; this.rqueueWebConfig = rqueueWebConfig; - this.rqueueMessageTemplate = rqueueMessageTemplate; + this.broker = broker; this.taskExecutionBackoff = taskExecutionBackoff; this.messageProcessorHandler = messageProcessorHandler; this.rqueueSystemConfigDao = rqueueSystemConfigDao; @@ -106,8 +105,7 @@ private void handleOldMessage(JobImpl job, RqueueMessage rqueueMessage) { null, rqueueMessage, job.getQueueDetail().getName()); - rqueueMessageTemplate.removeElementFromZset( - job.getQueueDetail().getProcessingQueueName(), rqueueMessage); + broker.ack(job.getQueueDetail(), rqueueMessage); } private void publishEvent(JobImpl job, RqueueMessage rqueueMessage, MessageStatus messageStatus) { @@ -125,38 +123,12 @@ private void updateMetadata( private void deleteMessage(JobImpl job, MessageStatus status, int failureCount) { RqueueMessage rqueueMessage = job.getRqueueMessage(); - rqueueMessageTemplate.removeElementFromZset( - job.getQueueDetail().getProcessingQueueName(), rqueueMessage); + broker.ack(job.getQueueDetail(), rqueueMessage); rqueueMessage.setFailureCount(failureCount); messageProcessorHandler.handleMessage(job, status); publishEvent(job, job.getRqueueMessage(), status); } - private void moveMessageToQueue( - QueueDetail queueDetail, - String queueName, - RqueueMessage oldMessage, - RqueueMessage newMessage, - long delay) { - RedisUtils.executePipeLine( - rqueueMessageTemplate.getTemplate(), (connection, keySerializer, valueSerializer) -> { - byte[] newMessageBytes = valueSerializer.serialize(newMessage); - byte[] oldMessageBytes = valueSerializer.serialize(oldMessage); - byte[] processingQueueNameBytes = - keySerializer.serialize(queueDetail.getProcessingQueueName()); - byte[] queueNameBytes = keySerializer.serialize(queueName); - assert queueNameBytes != null; - assert newMessageBytes != null; - if (delay > 0) { - connection.zAdd(queueNameBytes, delay, newMessageBytes); - } else { - connection.lPush(queueNameBytes, newMessageBytes); - } - assert processingQueueNameBytes != null; - connection.zRem(processingQueueNameBytes, oldMessageBytes); - }); - } - private void moveMessageToDlq(JobImpl job, int failureCount, Throwable throwable) { log( Level.DEBUG, @@ -180,11 +152,9 @@ private void moveMessageToDlq(JobImpl job, int failureCount, Throwable throwable "Queue Config not found for queue {}", null, queueDetail.getDeadLetterQueue()); - moveMessageToQueue( + broker.moveToDlq( queueDetail, queueDetail.getDeadLetterQueueName(), rqueueMessage, newMessage, -1); } else { - // update queue name to dead letter queue - // task execution backoff should consider this to identify if it's part of dead letter queue newMessage.setQueueName(queueConfig.getName()); newMessage.setFailureCount(0); newMessage.setSourceQueueName(rqueueMessage.getQueueName()); @@ -193,11 +163,11 @@ private void moveMessageToDlq(JobImpl job, int failureCount, Throwable throwable backOff = (backOff == TaskExecutionBackOff.STOP) ? FixedTaskExecutionBackOff.DEFAULT_INTERVAL : backOff; - moveMessageToQueue( + broker.moveToDlq( queueDetail, queueConfig.getScheduledQueueName(), rqueueMessage, newMessage, backOff); } } else { - moveMessageToQueue( + broker.moveToDlq( queueDetail, queueDetail.getDeadLetterQueueName(), rqueueMessage, newMessage, -1); } publishEvent(job, newMessage, MessageStatus.MOVED_TO_DLQ); @@ -207,20 +177,7 @@ RqueueMessage parkMessageForRetry( RqueueMessage rqueueMessage, int failureCount, long delay, QueueDetail queueDetail) { RqueueMessage newMessage = rqueueMessage.toBuilder().failureCount(failureCount).build().updateReEnqueuedAt(); - if (delay <= 0) { - rqueueMessageTemplate.moveMessage( - queueDetail.getProcessingQueueName(), - queueDetail.getQueueName(), - rqueueMessage, - newMessage); - } else { - rqueueMessageTemplate.moveMessageWithDelay( - queueDetail.getProcessingQueueName(), - queueDetail.getScheduledQueueName(), - rqueueMessage, - newMessage, - delay); - } + broker.parkForRetry(queueDetail, rqueueMessage, newMessage, delay); return newMessage; } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/QueueDetail.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/QueueDetail.java index a57f2e2ef..9cd159c42 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/QueueDetail.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/QueueDetail.java @@ -17,6 +17,7 @@ package com.github.sonus21.rqueue.listener; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.github.sonus21.rqueue.enums.QueueType; import com.github.sonus21.rqueue.models.Concurrency; import com.github.sonus21.rqueue.models.SerializableBase; import com.github.sonus21.rqueue.models.db.DeadLetterQueue; @@ -41,6 +42,17 @@ import lombok.ToString; import org.springframework.util.CollectionUtils; +/** + * Configuration and metadata for an Rqueue listener. Each queue detail captures the queue's + * polling behavior, error handling, and optional dead-letter queue setup. + * + *

Internal field names (e.g., {@code queueName}, {@code processingQueueName}) are backed by the + * queue name and prefixes from {@link com.github.sonus21.rqueue.config.RqueueConfig}. + * + *

Supports priority-based sub-queues via the {@code priority} map, where each entry defines a + * priority level and its relative weight. Use {@code expandQueueDetail()} to generate concrete + * queue details for each priority when multi-priority queueing is configured. + */ @Getter @Builder @EqualsAndHashCode(callSuper = true) @@ -72,6 +84,8 @@ public class QueueDetail extends SerializableBase { private String priorityGroup; private Set> doNotRetry; + private final String consumerName; + public boolean isDlqSet() { return !StringUtils.isEmpty(deadLetterQueueName); } @@ -151,6 +165,7 @@ private QueueDetail cloneQueueDetail( .concurrency(concurrency) .priority(Collections.singletonMap(Constants.DEFAULT_PRIORITY_KEY, priority)) .doNotRetry(doNotRetry) + .consumerName(consumerName) .build(); } @@ -158,9 +173,20 @@ public Duration visibilityDuration() { return Duration.ofMillis(visibilityTimeout); } - public enum QueueType { - QUEUE, - STREAM + /** + * Returns the effective JetStream consumer name for this queue. When {@link #consumerName} is + * explicitly set it is returned as-is. Otherwise a default is derived from the queue name: + * primary (non-system-generated) queues get {@code {name}-consumer-primary}; system-generated + * priority sub-queues get {@code {name}-consumer}. The name is sanitized so that characters + * outside {@code [A-Za-z0-9_-]} (e.g. the {@code ::} priority suffix separator) are replaced + * with {@code -}, producing a valid NATS consumer name in all cases. + */ + public String resolvedConsumerName() { + if (consumerName != null && !consumerName.isEmpty()) { + return consumerName; + } + String sanitized = name.replaceAll("[^A-Za-z0-9_-]", "-"); + return systemGenerated ? sanitized + "-consumer" : sanitized + "-consumer-primary"; } public boolean isDoNotRetryError(Throwable throwable) { diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java index ed850bd30..bbcb4acbc 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java @@ -98,7 +98,7 @@ private void init() { beanProvider.getRqueueConfig(), beanProvider.getRqueueMessageMetadataService(), beanProvider.getRqueueJobDao(), - beanProvider.getRqueueMessageTemplate(), + beanProvider.getMessageBroker(), beanProvider.getRqueueLockManager(), queueDetail, messageMetadata, @@ -344,18 +344,10 @@ private void schedulePeriodicMessage() { .build(); String messageKey = getScheduledMessageKey(newMessage); long expiryInSeconds = getTtlForScheduledMessageKey(newMessage); - log( - Level.DEBUG, - "Schedule periodic message: {} Status: {}", - null, - job.getRqueueMessage(), - beanProvider - .getRqueueMessageTemplate() - .scheduleMessage( - job.getQueueDetail().getScheduledQueueName(), - messageKey, - newMessage, - expiryInSeconds)); + log(Level.DEBUG, "Schedule periodic message: {}", null, job.getRqueueMessage()); + beanProvider + .getMessageBroker() + .scheduleNext(job.getQueueDetail(), messageKey, newMessage, expiryInSeconds); } private void handlePeriodicMessage() { diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageHandler.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageHandler.java index 510e93b65..67797d061 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageHandler.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageHandler.java @@ -78,6 +78,18 @@ import org.springframework.util.MultiValueMap; import org.springframework.util.comparator.ComparableComparator; +/** + * Invokes {@link RqueueListener}-annotated listener methods and routes exceptions to + * {@link RqueueHandler}-annotated error handlers. + * + *

Extends Spring's {@link AbstractMethodMessageHandler} to discover listener methods via + * reflection, deserialize incoming messages to the correct argument types, and dispatch to the + * appropriate handler based on message content and method signature. + * + *

Supports multiple argument injection patterns: message payload (with type conversion), raw + * {@code Message} envelopes, Spring headers, and custom parameters via {@code @Header}. + * Exception handlers are matched by exception type with fallback to base exception class. + */ @Slf4j public class RqueueMessageHandler extends AbstractMethodMessageHandler { @@ -92,6 +104,12 @@ public class RqueueMessageHandler extends AbstractMethodMessageHandler handlerMethods = new LinkedMultiValueMap<>(64); + // Phase 3: when set to false (NATS-style brokers), the "exactly one primary per queue" + // validation in afterPropertiesSet() is skipped and a single WARN is logged listing queues + // with multiple handlers. Defaults to true to preserve Redis semantics. + private boolean primaryHandlerDispatchEnabled = true; + private static volatile boolean rqueueHandlerWarnLogged = false; + public RqueueMessageHandler(final MessageConverter messageConverter, boolean inspectAllBean) { notNull(messageConverter, "messageConverter cannot be null"); this.messageConverter = messageConverter; @@ -136,9 +154,44 @@ protected List initReturnValueHandler return new ArrayList<>(getCustomReturnValueHandlers()); } + /** + * Capability flag set by the listener container when a non-Redis broker is active. When + * {@code false}, the {@code @RqueueHandler} primary validation is skipped because secondary + * handler dispatch is not honored by such brokers. + */ + public void setPrimaryHandlerDispatchEnabled(boolean primaryHandlerDispatchEnabled) { + this.primaryHandlerDispatchEnabled = primaryHandlerDispatchEnabled; + } + + public boolean isPrimaryHandlerDispatchEnabled() { + return primaryHandlerDispatchEnabled; + } + @Override public void afterPropertiesSet() { super.afterPropertiesSet(); + if (!primaryHandlerDispatchEnabled) { + // NATS-style backend: secondary handler dispatch is not supported; warn once with the + // queues that have multiple handler methods so users can convert them to listeners. + if (!rqueueHandlerWarnLogged) { + for (Entry e : destinationLookup.entrySet()) { + List methods = handlerMethods.get(e.getValue()); + if (methods != null && methods.size() > 1) { + int secondary = (int) methods.stream().filter(m -> !m.primary).count(); + if (secondary > 0) { + log.warn( + "@RqueueHandler is not honored by the NATS backend; the {} secondary handler " + + "methods on queue '{}' will not receive messages. Convert them to " + + "@RqueueListener with their own consumerName.", + secondary, + e.getKey()); + rqueueHandlerWarnLogged = true; + } + } + } + } + return; + } for (Entry e : destinationLookup.entrySet()) { List handlerMethodWithPrimaries = handlerMethods.get(e.getValue()); if (handlerMethodWithPrimaries.size() > 1) { @@ -211,9 +264,15 @@ protected void registerHandlerMethod(Object handler, Method method, MappingInfor @Override protected void handleMessageInternal(Message message, String lookupDestination) { - MappingInformation mapping = this.destinationLookup.get(lookupDestination); + String consumerName = (String) message.getHeaders().get(RqueueMessageHeaders.CONSUMER_NAME); + String lookupKey = (consumerName != null && !consumerName.isEmpty()) + ? consumerLookupKey(lookupDestination, consumerName) + : lookupDestination; + MappingInformation mapping = this.destinationLookup.get(lookupKey); Set matches = new HashSet<>(); - addMatchesToCollection(mapping, message, matches); + if (mapping != null) { + addMatchesToCollection(mapping, message, matches); + } if (matches.isEmpty()) { handleNoMatch(this.getHandlerMethodMap().keySet(), lookupDestination, message); return; @@ -272,6 +331,8 @@ private MappingInformation getMappingInformation(RqueueListener rqueueListener) Map priorityMap = resolvePriority(rqueueListener); String priorityGroup = resolvePriorityGroup(rqueueListener); int batchSize = getBatchSize(rqueueListener, concurrency); + String natsConsumerName = + ValueResolver.resolveKeyToString(getApplicationContext(), rqueueListener.consumerName()); MappingInformation mappingInformation = MappingInformation.builder() .active(active) .concurrency(concurrency) @@ -284,6 +345,8 @@ private MappingInformation getMappingInformation(RqueueListener rqueueListener) .priority(priorityMap) .batchSize(batchSize) .doNotRetry(new HashSet<>(Arrays.asList(rqueueListener.doNotRetry()))) + .consumerName(natsConsumerName.isEmpty() ? null : natsConsumerName) + .queueType(rqueueListener.mode()) .build(); if (mappingInformation.isValid()) { return mappingInformation; @@ -479,13 +542,25 @@ private Set resolveQueueNames(RqueueListener rqueueListener) { @Override protected Set getDirectLookupDestinations(MappingInformation mapping) { - Set destinations = new HashSet<>(mapping.getQueueNames()); + String consumerName = mapping.getConsumerName(); + Set destinations = new HashSet<>(); for (String queueName : mapping.getQueueNames()) { - destinations.addAll(PriorityUtils.getNamesFromPriority(queueName, mapping.getPriority())); + if (consumerName != null && !consumerName.isEmpty()) { + // Consumer-qualified key so two @RqueueListener methods with different consumerNames + // on the same queue get independent destinationLookup entries. + destinations.add(consumerLookupKey(queueName, consumerName)); + } else { + destinations.add(queueName); + destinations.addAll(PriorityUtils.getNamesFromPriority(queueName, mapping.getPriority())); + } } return destinations; } + static String consumerLookupKey(String queueName, String consumerName) { + return queueName + "##" + consumerName; + } + @Override protected String getDestination(Message message) { return (String) message.getHeaders().get(RqueueMessageHeaders.DESTINATION); diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageHeaders.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageHeaders.java index 33f0f3953..f3729b77f 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageHeaders.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageHeaders.java @@ -56,6 +56,13 @@ public final class RqueueMessageHeaders { */ public static final String EXECUTION = "execution"; + /** + * NATS JetStream consumer name — set when a message was fetched for a specific named consumer so + * that {@link RqueueMessageHandler} can route it to the correct {@code @RqueueListener} method. + * Absent (null) for single-consumer queues and all Redis-backed queues. + */ + public static final String CONSUMER_NAME = "consumerName"; + private static final MessageHeaders emptyMessageHeaders = new MessageHeaders(Collections.emptyMap()); @@ -71,8 +78,21 @@ public static MessageHeaders buildMessageHeaders( Job job, Execution execution, MessageHeaders messageHeaders) { - Map headers = new HashMap<>(9); + return buildMessageHeaders(destination, null, rqueueMessage, job, execution, messageHeaders); + } + + public static MessageHeaders buildMessageHeaders( + String destination, + String consumerName, + RqueueMessage rqueueMessage, + Job job, + Execution execution, + MessageHeaders messageHeaders) { + Map headers = new HashMap<>(10); headers.put(DESTINATION, destination); + if (consumerName != null && !consumerName.isEmpty()) { + headers.put(CONSUMER_NAME, consumerName); + } headers.put(ID, rqueueMessage.getId()); headers.put(MESSAGE, rqueueMessage); if (job != null) { diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageListenerContainer.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageListenerContainer.java index 3922c6f2b..b680fa6e5 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageListenerContainer.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageListenerContainer.java @@ -27,6 +27,8 @@ import com.github.sonus21.rqueue.core.RqueueBeanProvider; import com.github.sonus21.rqueue.core.RqueueMessageTemplate; import com.github.sonus21.rqueue.core.middleware.Middleware; +import com.github.sonus21.rqueue.core.spi.MessageBroker; +import com.github.sonus21.rqueue.core.spi.redis.RedisMessageBroker; import com.github.sonus21.rqueue.core.support.MessageProcessor; import com.github.sonus21.rqueue.models.Concurrency; import com.github.sonus21.rqueue.models.db.QueueConfig; @@ -76,6 +78,7 @@ public class RqueueMessageListenerContainer public static final String EVENT_SOURCE = "RqueueMessageListenerContainer"; private static final String DEFAULT_THREAD_NAME_PREFIX = ClassUtils.getShortName(RqueueMessageListenerContainer.class); + static final String POLLER_KEY_SEP = "##"; final QueueStateMgr queueStateMgr = new QueueStateMgr(); private final Object lifecycleMgr = new Object(); private final RqueueMessageTemplate rqueueMessageTemplate; @@ -84,6 +87,9 @@ public class RqueueMessageListenerContainer private final ConcurrentHashMap> scheduledFutureByQueue = new ConcurrentHashMap<>(); private final Map queueThreadMap = new ConcurrentHashMap<>(); + // tracks which (queue, consumer) pollers have been started; prevents the second consumer of a + // multi-consumer queue from being skipped by the bare-name dedup check in startQueue. + private final java.util.Set startedPollerKeys = ConcurrentHashMap.newKeySet(); @Autowired protected RqueueBeanProvider rqueueBeanProvider; @@ -109,6 +115,10 @@ public class RqueueMessageListenerContainer private PriorityMode priorityMode; private MessageHeaders messageHeaders; private HardStrictPriorityPollerProperties hardStrictPriorityPollerProperties; + // Optional pluggable backend. Additive in Phase 1: when null (current code path), behavior is + // unchanged. When set, capabilities() may gate scheduler/handler dispatch wiring; the Redis + // broker reports REDIS_DEFAULTS so the gated branches still match existing semantics. + private MessageBroker messageBroker; public RqueueMessageListenerContainer( RqueueMessageHandler rqueueMessageHandler, RqueueMessageTemplate rqueueMessageTemplate) { @@ -148,6 +158,42 @@ public RqueueMessageHandler getRqueueMessageHandler() { return rqueueMessageHandler; } + /** + * Returns the optional {@link MessageBroker} backing this container, or {@code null} if the + * container is using the legacy Redis-backed code path. + */ + public MessageBroker getMessageBroker() { + return messageBroker; + } + + /** + * Optionally set a {@link MessageBroker}. Additive in Phase 1: when not set (default), the + * container behaves exactly as before. When set, capability flags may gate scheduler and handler + * dispatch wiring. The default Redis broker reports {@code REDIS_DEFAULTS} so all gated branches + * resolve to the existing behavior. + * + * @param messageBroker broker SPI instance, or {@code null} + */ + public void setMessageBroker(MessageBroker messageBroker) { + this.messageBroker = messageBroker; + } + + /** + * Whether scheduler startup should run for this container. Returns {@code true} for the legacy + * (no broker) path and for brokers that advertise scheduled introspection support. + */ + public boolean isScheduledIntrospectionEnabled() { + return messageBroker == null || messageBroker.capabilities().supportsScheduledIntrospection(); + } + + /** + * Whether the primary {@link RqueueMessageHandler} dispatch path is enabled. Returns {@code true} + * for the legacy (no broker) path and for brokers that advertise primary handler dispatch. + */ + public boolean isPrimaryHandlerDispatchEnabled() { + return messageBroker == null || messageBroker.capabilities().usesPrimaryHandlerDispatch(); + } + public Integer getMaxNumWorkers() { return maxNumWorkers; } @@ -190,6 +236,13 @@ protected void doDestroy() { executor.destroy(); } } + if (messageBroker instanceof AutoCloseable) { + try { + ((AutoCloseable) messageBroker).close(); + } catch (Exception e) { + log.warn("Error closing MessageBroker on destroy: {}", e.toString(), e); + } + } } @Override @@ -241,9 +294,15 @@ public void stop(Runnable callback) { private void initializeQueueRegistry() { log.info("Initializing queue registry"); EndpointRegistry.delete(); + // Validate every @RqueueListener queue name against backend rules before registering, so + // illegal names (e.g. dots when running on NATS) fail loudly at startup instead of surfacing + // as opaque driver errors at first publish. for (MappingInformation mappingInformation : rqueueMessageHandler.getHandlerMethodMap().keySet()) { for (String queue : mappingInformation.getQueueNames()) { + if (messageBroker != null) { + messageBroker.validateQueueName(queue); + } for (QueueDetail queueDetail : getQueueDetail(queue, mappingInformation)) { EndpointRegistry.register(queueDetail); } @@ -283,17 +342,24 @@ private void initializeThreadMapForNonDefaultExecutor( } private void initialize() { + // Resolve the effective broker: injected (e.g. NATS) or a Redis wrapper for the legacy path. + MessageBroker effectiveBroker = + messageBroker != null ? messageBroker : new RedisMessageBroker(rqueueMessageTemplate); + rqueueBeanProvider.setMessageBroker(effectiveBroker); + + MessageProcessorHandler msgProcessorHandler = new MessageProcessorHandler( + manualDeletionMessageProcessor, + deadLetterQueueMessageProcessor, + discardMessageProcessor, + postExecutionMessageProcessor); + initializeQueue(); this.postProcessingHandler = new PostProcessingHandler( rqueueBeanProvider.getRqueueWebConfig(), rqueueBeanProvider.getApplicationEventPublisher(), - rqueueMessageTemplate, + effectiveBroker, taskExecutionBackOff, - new MessageProcessorHandler( - manualDeletionMessageProcessor, - deadLetterQueueMessageProcessor, - discardMessageProcessor, - postExecutionMessageProcessor), + msgProcessorHandler, rqueueBeanProvider.getRqueueSystemConfigDao()); this.rqueueBeanProvider.setPreExecutionMessageProcessor(preExecutionMessageProcessor); } @@ -301,6 +367,11 @@ private void initialize() { @Override public void afterPropertiesSet() throws Exception { synchronized (lifecycleMgr) { + // Propagate broker capability into the handler before its afterPropertiesSet runs the + // primary-validation. The handler is wired through Spring lifecycle separately, but if + // it's already been initialized this is a no-op for the primary path; the handler will + // re-check its flag when validating. + rqueueMessageHandler.setPrimaryHandlerDispatchEnabled(isPrimaryHandlerDispatchEnabled()); RqueueConfig rqueueConfig = rqueueBeanProvider.getRqueueConfig(); initializeQueueRegistry(); if (rqueueConfig.isProducer()) { @@ -332,6 +403,11 @@ private void initializeRunningQueueState() { } } + static String pollerKey(QueueDetail qd) { + String cn = qd.getConsumerName(); + return (cn != null && !cn.isEmpty()) ? qd.getName() + POLLER_KEY_SEP + cn : qd.getName(); + } + private int getWorkersCount(int queueCount) { return (maxNumWorkers == null ? queueCount * DEFAULT_WORKER_COUNT_PER_QUEUE : maxNumWorkers); } @@ -427,6 +503,8 @@ private List getQueueDetail(String queue, MappingInformation mappin .priority(priority) .priorityGroup(priorityGroup) .doNotRetry(mappingInformation.getDoNotRetry()) + .consumerName(mappingInformation.getConsumerName()) + .type(mappingInformation.getQueueType()) .build(); List queueDetails; if (queueDetail.getPriority().size() <= 1) { @@ -460,7 +538,7 @@ protected void doStart() { for (QueueDetail queueDetail : EndpointRegistry.getActiveQueueDetails()) { int prioritySize = queueDetail.getPriority().size(); if (prioritySize == 0) { - startQueue(queueDetail.getName(), queueDetail); + startQueue(pollerKey(queueDetail), queueDetail); } else { List queueDetails = queueGroupToDetails.getOrDefault(queueDetail.getPriorityGroup(), new ArrayList<>()); @@ -541,15 +619,17 @@ protected void startGroup(String groupName, List queueDetails) { scheduledFutureByQueue.put(groupName, future); } - protected void startQueue(String queueName, QueueDetail queueDetail) { - if (Boolean.TRUE.equals(queueRunningState.get(queueName))) { - return; + protected void startQueue(String pollerKey, QueueDetail queueDetail) { + if (!startedPollerKeys.add(pollerKey)) { + return; // already started (dedup for multi-consumer queues) } + String queueName = queueDetail.getName(); queueRunningState.put(queueName, true); QueueConfig config = rqueueBeanProvider.getRqueueSystemConfigDao().getConfigByName(queueName); queueStateMgr.pauseQueueIfRequired(config); QueueThreadPool queueThreadPool = queueThreadMap.get(queueName); DefaultRqueuePoller messagePoller = new DefaultRqueuePoller( + queueName, queueDetail, queueThreadPool, rqueueBeanProvider, @@ -560,7 +640,7 @@ protected void startQueue(String queueName, QueueDetail queueDetail) { postProcessingHandler, getMessageHeaders()); Future future = getTaskExecutor().submit(messagePoller); - scheduledFutureByQueue.put(queueName, future); + scheduledFutureByQueue.put(pollerKey, future); } public void pauseUnpauseQueue(String queue, boolean pause) { @@ -600,18 +680,17 @@ protected void doStop() { } } waitForRunningQueuesToStop(); + startedPollerKeys.clear(); } private void waitForRunningQueuesToStop() { - for (Map.Entry entry : queueRunningState.entrySet()) { - String queueName = entry.getKey(); - Future queueSpinningThread = scheduledFutureByQueue.get(queueName); + for (Map.Entry> entry : scheduledFutureByQueue.entrySet()) { waitForTermination( log, - queueSpinningThread, + entry.getValue(), getMaxWorkerWaitTime(), "An exception occurred while stopping queue '{}'", - queueName); + entry.getKey()); } if (!waitForWorkerTermination(queueThreadMap.values(), getMaxWorkerWaitTime())) { log.error("Some workers are not stopped within time"); diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessagePoller.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessagePoller.java index c6b483ea4..dfdddf92f 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessagePoller.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessagePoller.java @@ -22,6 +22,7 @@ import com.github.sonus21.rqueue.listener.RqueueMessageListenerContainer.QueueStateMgr; import com.github.sonus21.rqueue.utils.Constants; import com.github.sonus21.rqueue.utils.QueueThreadPool; +import java.time.Duration; import java.util.List; import org.slf4j.LoggerFactory; import org.slf4j.event.Level; @@ -59,13 +60,12 @@ abstract class RqueueMessagePoller extends MessageContainerBase { private List getMessages(QueueDetail queueDetail, int count) { return rqueueBeanProvider - .getRqueueMessageTemplate() + .getMessageBroker() .pop( - queueDetail.getQueueName(), - queueDetail.getProcessingQueueName(), - queueDetail.getProcessingQueueChannelName(), - queueDetail.getVisibilityTimeout(), - count); + queueDetail, + queueDetail.resolvedConsumerName(), + count, + Duration.ofMillis(pollingInterval)); } private void execute( diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/metrics/RqueueMetrics.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/metrics/RqueueMetrics.java index d4389925e..f862e8cc4 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/metrics/RqueueMetrics.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/metrics/RqueueMetrics.java @@ -18,7 +18,6 @@ import com.github.sonus21.rqueue.config.MetricsProperties; import com.github.sonus21.rqueue.core.EndpointRegistry; -import com.github.sonus21.rqueue.dao.RqueueStringDao; import com.github.sonus21.rqueue.listener.QueueDetail; import com.github.sonus21.rqueue.models.event.RqueueBootstrapEvent; import io.micrometer.core.instrument.Gauge; @@ -49,25 +48,12 @@ public class RqueueMetrics implements RqueueMetricsRegistry { private MeterRegistry meterRegistry; @Autowired - private RqueueStringDao rqueueStringDao; + private RqueueQueueMetricsProvider queueMetricsProvider; public RqueueMetrics(QueueCounter queueCounter) { this.queueCounter = queueCounter; } - private long size(String name, boolean isZset) { - Long val; - if (!isZset) { - val = rqueueStringDao.getListSize(name); - } else { - val = rqueueStringDao.getSortedSetSize(name); - } - if (val == null) { - return 0; - } - return val; - } - private void monitor() { for (QueueDetail queueDetail : EndpointRegistry.getActiveQueueDetails()) { Tags queueTags = @@ -75,21 +61,21 @@ private void monitor() { Gauge.builder( metricsProperties.getMetricName(QUEUE_SIZE), queueDetail, - c -> size(queueDetail.getQueueName(), false)) + c -> queueMetricsProvider.getPendingMessageCount(queueDetail.getName())) .tags(queueTags.and(QUEUE_KEY, queueDetail.getQueueName())) .description("The number of entries in this queue") .register(meterRegistry); Gauge.builder( metricsProperties.getMetricName(PROCESSING_QUEUE_SIZE), queueDetail, - c -> size(queueDetail.getProcessingQueueName(), true)) + c -> queueMetricsProvider.getProcessingMessageCount(queueDetail.getName())) .tags(queueTags.and(QUEUE_KEY, queueDetail.getProcessingQueueName())) .description("The number of entries in the processing queue") .register(meterRegistry); Gauge.builder( metricsProperties.getMetricName(SCHEDULED_QUEUE_SIZE), queueDetail, - c -> size(queueDetail.getScheduledQueueName(), true)) + c -> queueMetricsProvider.getScheduledMessageCount(queueDetail.getName())) .tags(queueTags.and(QUEUE_KEY, queueDetail.getScheduledQueueName())) .description("The number of entries waiting in the scheduled queue") .register(meterRegistry); @@ -97,7 +83,7 @@ private void monitor() { Builder builder = Gauge.builder( metricsProperties.getMetricName(DEAD_LETTER_QUEUE_SIZE), queueDetail, - c -> size(queueDetail.getDeadLetterQueueName(), false)); + c -> queueMetricsProvider.getDeadLetterMessageCount(queueDetail.getName())); builder.tags(queueTags); builder.description("The number of entries in the dead letter queue"); builder.register(meterRegistry); diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/metrics/RqueueQueueMetrics.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/metrics/RqueueQueueMetrics.java deleted file mode 100644 index 982c2bce5..000000000 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/metrics/RqueueQueueMetrics.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright (c) 2021-2026 Sonu Kumar - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * You may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and limitations under the License. - * - */ - -package com.github.sonus21.rqueue.metrics; - -import com.github.sonus21.rqueue.common.RqueueRedisTemplate; -import com.github.sonus21.rqueue.core.EndpointRegistry; -import com.github.sonus21.rqueue.exception.QueueDoesNotExist; -import com.github.sonus21.rqueue.listener.QueueDetail; - -/** - * This class reports queue message counter. - * - *

Count can be sent to some monitoring tool like Prometheus, influx db etc - */ -public class RqueueQueueMetrics { - - private final RqueueRedisTemplate redisTemplate; - - public RqueueQueueMetrics(RqueueRedisTemplate redisTemplate) { - this.redisTemplate = redisTemplate; - } - - /** - * Get number of messages waiting for consumption - * - * @param queue queue name - * @return -1 if queue is not registered otherwise message count - */ - public long getPendingMessageCount(String queue) { - try { - QueueDetail queueDetail = EndpointRegistry.get(queue); - return redisTemplate.getListSize(queueDetail.getQueueName()); - } catch (QueueDoesNotExist e) { - return -1; - } - } - - /** - * Get number of messages waiting in scheduled queue, these messages would move to pending queue - * as soon as the scheduled time is reach. - * - * @param queue queue name - * @return -1 if queue is not registered otherwise message count - */ - public long getScheduledMessageCount(String queue) { - try { - QueueDetail queueDetail = EndpointRegistry.get(queue); - return redisTemplate.getZsetSize(queueDetail.getScheduledQueueName()); - } catch (QueueDoesNotExist e) { - return -1; - } - } - - /** - * Get number of messages those are currently being processed - * - * @param queue queue name - * @return -1 if queue is not registered otherwise message count - */ - public long getProcessingMessageCount(String queue) { - try { - QueueDetail queueDetail = EndpointRegistry.get(queue); - return redisTemplate.getZsetSize(queueDetail.getProcessingQueueName()); - } catch (QueueDoesNotExist e) { - return -1; - } - } - - /** - * Get number of messages waiting for consumption - * - * @param queue queue name - * @param priority priority of this queue - * @return -1 if queue is not registered otherwise message count - */ - public long getPendingMessageCount(String queue, String priority) { - try { - QueueDetail queueDetail = EndpointRegistry.get(queue, priority); - return redisTemplate.getListSize(queueDetail.getQueueName()); - } catch (QueueDoesNotExist e) { - return -1; - } - } - - /** - * Get number of messages waiting in scheduled queue, these messages would move to pending queue - * as soon as the scheduled time is reach. - * - * @param queue queue name - * @param priority priority of this queue - * @return -1 if queue is not registered otherwise message count - */ - public long getScheduledMessageCount(String queue, String priority) { - try { - QueueDetail queueDetail = EndpointRegistry.get(queue, priority); - return redisTemplate.getZsetSize(queueDetail.getScheduledQueueName()); - } catch (QueueDoesNotExist e) { - return -1; - } - } - - /** - * Get number of messages those are currently being processed - * - * @param queue queue name - * @param priority priority of this queue - * @return -1 if queue is not registered otherwise message count - */ - public long getProcessingMessageCount(String queue, String priority) { - try { - QueueDetail queueDetail = EndpointRegistry.get(queue, priority); - return redisTemplate.getZsetSize(queueDetail.getProcessingQueueName()); - } catch (QueueDoesNotExist e) { - return -1; - } - } -} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/metrics/RqueueQueueMetricsProvider.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/metrics/RqueueQueueMetricsProvider.java new file mode 100644 index 000000000..f622541ca --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/metrics/RqueueQueueMetricsProvider.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ +package com.github.sonus21.rqueue.metrics; + +/** + * Backend-agnostic provider of queue-depth metrics. Each backend (Redis, NATS, ...) supplies its + * own implementation; consumers like {@link RqueueMetrics} read sizes through this interface + * instead of reaching into a Redis-shaped DAO. + * + *

The {@code queueName} argument is the user-facing queue name (the value bound on + * {@code @RqueueListener(value="...")}), not an internal storage key. Implementations are + * responsible for mapping it to the appropriate backend-specific key(s). + * + *

Implementations must return {@code 0} when a queue has no messages of the requested kind + * (rather than throwing) so callers can use the values directly as gauge readings. + */ +public interface RqueueQueueMetricsProvider { + + /** + * Number of messages waiting to be consumed from {@code queueName} — i.e. enqueued and ready for + * a worker to pick up, excluding messages already in-flight (processing) or scheduled for a + * future delivery time. + */ + long getPendingMessageCount(String queueName); + + /** + * Number of messages enqueued to {@code queueName} with a future delivery time that has not yet + * elapsed. Backends that don't support delayed delivery return {@code 0}. + */ + long getScheduledMessageCount(String queueName); + + /** + * Number of messages currently in-flight for {@code queueName} — handed to a worker but not yet + * acked or nacked. Backends without an explicit in-flight set return {@code 0}. + */ + long getProcessingMessageCount(String queueName); + + /** + * Number of messages in the dead-letter queue associated with {@code queueName}. Returns + * {@code 0} when no DLQ is configured for the queue or the backend does not surface DLQ depth. + */ + long getDeadLetterMessageCount(String queueName); + + /** + * Priority-aware variant of {@link #getPendingMessageCount(String)}. The default implementation + * ignores priority and returns the parent queue depth, which is the right behaviour for backends + * that don't model per-priority sub-queues. + */ + default long getPendingMessageCount(String queueName, String priority) { + return getPendingMessageCount(queueName); + } + + /** Priority-aware variant of {@link #getScheduledMessageCount(String)}. */ + default long getScheduledMessageCount(String queueName, String priority) { + return getScheduledMessageCount(queueName); + } + + /** Priority-aware variant of {@link #getProcessingMessageCount(String)}. */ + default long getProcessingMessageCount(String queueName, String priority) { + return getProcessingMessageCount(queueName); + } +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/event/RqueuePubSubEvent.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/event/RqueuePubSubEvent.java index ab9749e12..c91f843ba 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/event/RqueuePubSubEvent.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/event/RqueuePubSubEvent.java @@ -17,9 +17,9 @@ package com.github.sonus21.rqueue.models.event; import com.fasterxml.jackson.annotation.JsonIgnore; -import com.github.sonus21.rqueue.converter.GenericMessageConverter.SmartMessageSerDes; import com.github.sonus21.rqueue.models.SerializableBase; import com.github.sonus21.rqueue.models.enums.PubSubType; +import com.github.sonus21.rqueue.serdes.RqueueSerDes; import java.nio.charset.StandardCharsets; import lombok.AllArgsConstructor; import lombok.EqualsAndHashCode; @@ -41,7 +41,7 @@ public class RqueuePubSubEvent extends SerializableBase { private String message; @JsonIgnore - public T messageAs(SmartMessageSerDes serDes, Class clazz) { + public T messageAs(RqueueSerDes serDes, Class clazz) throws Exception { return serDes.deserialize(getMessage().getBytes(StandardCharsets.UTF_8), clazz); } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/registry/RqueueWorkerPollerMetadata.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/registry/RqueueWorkerPollerMetadata.java index 4304ffa17..09943c4d2 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/registry/RqueueWorkerPollerMetadata.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/registry/RqueueWorkerPollerMetadata.java @@ -37,6 +37,7 @@ public class RqueueWorkerPollerMetadata extends SerializableBase { private static final long serialVersionUID = -8612115593908071754L; private String workerId; + private String consumerName; private long lastPollAt; private Long lastMessageAt; private Long lastCapacityExhaustedAt; diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/registry/RqueueWorkerPollerView.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/registry/RqueueWorkerPollerView.java index d38605f16..c15a16a6a 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/registry/RqueueWorkerPollerView.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/registry/RqueueWorkerPollerView.java @@ -38,6 +38,7 @@ public class RqueueWorkerPollerView extends SerializableBase { private String queue; private String workerId; + private String consumerName; private String host; private String pid; private String status; diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/response/DataViewResponse.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/response/DataViewResponse.java index c642ea17f..97ed4dee5 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/response/DataViewResponse.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/response/DataViewResponse.java @@ -39,6 +39,23 @@ public class DataViewResponse extends BaseResponse { private List actions = new ArrayList<>(); private List rows = new LinkedList<>(); + /** + * Optional UI hint set by the dashboard service when the active {@code MessageBroker} + * does not support scheduled-queue introspection (see + * {@code Capabilities#supportsScheduledIntrospection}). When {@code true} the frontend + * should hide / grey out the scheduled-queue panel. Additive and defaults to {@code false} + * so the Redis backend's existing behavior is unchanged. + */ + private boolean hideScheduledPanel; + + /** + * Optional UI hint set by the dashboard service when the active {@code MessageBroker} + * does not support cron jobs (see {@code Capabilities#supportsCronJobs}). When {@code true} + * the frontend should hide / grey out cron-style management. Additive and defaults to + * {@code false}. + */ + private boolean hideCronJobs; + public static DataViewResponse createErrorMessage(String message) { DataViewResponse response = new DataViewResponse(); response.setCode(1); diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/repository/MessageBrowsingRepository.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/repository/MessageBrowsingRepository.java new file mode 100644 index 000000000..0e94dfefb --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/repository/MessageBrowsingRepository.java @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ + +package com.github.sonus21.rqueue.repository; + +import com.github.sonus21.rqueue.models.enums.DataType; +import com.github.sonus21.rqueue.models.response.DataViewResponse; +import java.util.List; + +/** + * Backend-shaped browsing primitives consumed by the dashboard's queue-detail and data-explorer + * pages. Replaces direct {@code RqueueRedisTemplate} access from + * {@code RqueueQDetailServiceImpl}, allowing a single backend-neutral service impl in + * {@code rqueue-web}. + * + *

The interface deliberately exposes Redis-shaped operations on arbitrary keys (LIST / ZSET / + * SET / KEY); it is the storage-layer abstraction, not a feature contract. Backends without an + * equivalent (e.g. NATS JetStream KV) throw {@code BackendCapabilityException} from + * {@link #viewData} and return {@code 0} from the size queries; the dashboard either hides the + * panel (via capability flags) or surfaces the 501 cleanly. + */ +public interface MessageBrowsingRepository { + + /** + * Size of a single data structure addressed by {@code name}. Returns {@code 0} when the key + * does not exist or the backend cannot model the structure. + */ + long getDataSize(String name, DataType type); + + /** + * Bulk size — same semantics as {@link #getDataSize} per element, but Redis impls are + * expected to pipeline the round-trips. {@code names} and {@code types} must be the same + * length; the returned list is the same length and order. + */ + List getDataSizes(List names, List types); + + /** + * Raw data-explorer browser used by the dashboard's "Data" page. Reads paginated rows out of + * a backing data structure (LIST / ZSET / SET) or returns a single keyed value (KEY / single + * ZSET member score). Backends without arbitrary keyed reads throw + * {@code BackendCapabilityException}. + * + * @param name the storage key to read. + * @param type the structure type at {@code name}. + * @param key optional sub-key (e.g. ZSET member name to fetch a single score). Empty / null + * returns a paginated range. + * @param pageNumber zero-based page index. + * @param itemPerPage page size. + */ + DataViewResponse viewData( + String name, DataType type, String key, int pageNumber, int itemPerPage); +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/serdes/JacksonTypeEnvelop.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/serdes/JacksonTypeEnvelop.java new file mode 100644 index 000000000..b6179ef4f --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/serdes/JacksonTypeEnvelop.java @@ -0,0 +1,12 @@ +package com.github.sonus21.rqueue.serdes; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import tools.jackson.databind.JavaType; + +@AllArgsConstructor +@Getter +public class JacksonTypeEnvelop implements TypeEnvelop { + + private final JavaType javaType; +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/serdes/RqJacksonSerDes.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/serdes/RqJacksonSerDes.java new file mode 100644 index 000000000..172c65ab9 --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/serdes/RqJacksonSerDes.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ +package com.github.sonus21.rqueue.serdes; + +import java.io.IOException; +import tools.jackson.databind.ObjectMapper; + +/** + * {@link RqueueSerDes} implementation backed by a Jackson {@link ObjectMapper}. + */ +public class RqJacksonSerDes implements RqueueSerDes { + + private final ObjectMapper mapper; + + public RqJacksonSerDes(ObjectMapper mapper) { + this.mapper = mapper; + } + + @Override + public String serializeAsString(T value) throws IOException { + return mapper.writeValueAsString(value); + } + + @Override + public byte[] serialize(T value) throws IOException { + return mapper.writeValueAsBytes(value); + } + + @Override + public T deserialize(byte[] bytes, Class klass) throws IOException { + return mapper.readValue(bytes, klass); + } + + @Override + public T deserialize(byte[] msg, TypeEnvelop type) { + return mapper.readValue(msg, ((JacksonTypeEnvelop) type).getJavaType()); + } + + @Override + public T deserialize(String msg, Class klass) throws IOException { + return mapper.readValue(msg, klass); + } +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/serdes/RqJacksonTypeFactory.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/serdes/RqJacksonTypeFactory.java new file mode 100644 index 000000000..3b4e8b4d9 --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/serdes/RqJacksonTypeFactory.java @@ -0,0 +1,25 @@ +package com.github.sonus21.rqueue.serdes; + +import tools.jackson.databind.JavaType; +import tools.jackson.databind.ObjectMapper; + +public class RqJacksonTypeFactory implements RqueueTypeFactory { + + private final ObjectMapper mapper; + + public RqJacksonTypeFactory(ObjectMapper mapper) { + this.mapper = mapper; + } + + @Override + public TypeEnvelop create(Class clazz) { + JavaType type = mapper.getTypeFactory().constructType(clazz); + return new JacksonTypeEnvelop(type); + } + + @Override + public TypeEnvelop create(Class parametrized, Class... parameterClasses) { + JavaType type = mapper.getTypeFactory().constructParametricType(parametrized, parameterClasses); + return new JacksonTypeEnvelop(type); + } +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/serdes/RqueueSerDes.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/serdes/RqueueSerDes.java new file mode 100644 index 000000000..898ece3c1 --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/serdes/RqueueSerDes.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ +package com.github.sonus21.rqueue.serdes; + +import java.io.IOException; + +/** + * Serialization / deserialization contract used by the NATS backend. + */ +public interface RqueueSerDes { + + byte[] serialize(T value) throws IOException; + + String serializeAsString(T value) throws IOException; + + T deserialize(byte[] bytes, Class klass) throws IOException; + + T deserialize(byte[] msg, TypeEnvelop type) throws IOException; + + T deserialize(String msg, Class klass) throws IOException; +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/serdes/RqueueTypeFactory.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/serdes/RqueueTypeFactory.java new file mode 100644 index 000000000..f33bfc6fe --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/serdes/RqueueTypeFactory.java @@ -0,0 +1,8 @@ +package com.github.sonus21.rqueue.serdes; + +public interface RqueueTypeFactory { + + TypeEnvelop create(Class clazz); + + TypeEnvelop create(Class parametrized, Class... parameterClasses); +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/SerializationUtils.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/serdes/SerializationUtils.java similarity index 69% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/SerializationUtils.java rename to rqueue-core/src/main/java/com/github/sonus21/rqueue/serdes/SerializationUtils.java index 74da045dc..fbdc8082e 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/SerializationUtils.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/serdes/SerializationUtils.java @@ -14,8 +14,10 @@ * */ -package com.github.sonus21.rqueue.utils; +package com.github.sonus21.rqueue.serdes; +import com.github.sonus21.rqueue.utils.StringUtils; +import lombok.Getter; import tools.jackson.databind.DeserializationFeature; import tools.jackson.databind.ObjectMapper; import tools.jackson.databind.json.JsonMapper; @@ -24,6 +26,16 @@ public final class SerializationUtils { public static final byte[] EMPTY_ARRAY = new byte[0]; + // These have been left public so callers can override if needed. + @Getter + public static ObjectMapper objectMapper = createObjectMapper(); + + @Getter + public static RqueueSerDes serDes = new RqJacksonSerDes(objectMapper); + + @Getter + public static RqueueTypeFactory typeFactory = new RqJacksonTypeFactory(objectMapper); + private SerializationUtils() {} public static boolean isEmpty(byte[] bytes) { @@ -36,9 +48,10 @@ public static boolean isJson(String data) { && data.charAt(data.length() - 1) == '}'; } - public static ObjectMapper createObjectMapper() { + private static ObjectMapper createObjectMapper() { return JsonMapper.builder() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + .configure(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES, false) .findAndAddModules() .build(); } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/serdes/TypeEnvelop.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/serdes/TypeEnvelop.java new file mode 100644 index 000000000..81f646b26 --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/serdes/TypeEnvelop.java @@ -0,0 +1,3 @@ +package com.github.sonus21.rqueue.serdes; + +public interface TypeEnvelop {} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueMessageMetadataService.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/service/RqueueMessageMetadataService.java similarity index 97% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueMessageMetadataService.java rename to rqueue-core/src/main/java/com/github/sonus21/rqueue/service/RqueueMessageMetadataService.java index e4921f5a2..8487469f6 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueMessageMetadataService.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/service/RqueueMessageMetadataService.java @@ -14,7 +14,7 @@ * */ -package com.github.sonus21.rqueue.web.service; +package com.github.sonus21.rqueue.service; import com.github.sonus21.rqueue.core.RqueueMessage; import com.github.sonus21.rqueue.models.db.MessageMetadata; diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueUtilityService.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/service/RqueueUtilityService.java similarity index 98% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueUtilityService.java rename to rqueue-core/src/main/java/com/github/sonus21/rqueue/service/RqueueUtilityService.java index 0dcba99f6..d1dc47895 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueUtilityService.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/service/RqueueUtilityService.java @@ -14,7 +14,7 @@ * */ -package com.github.sonus21.rqueue.web.service; +package com.github.sonus21.rqueue.service; import com.github.sonus21.rqueue.models.Pair; import com.github.sonus21.rqueue.models.enums.AggregationType; diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/HttpUtils.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/HttpUtils.java index c5f74bd09..34f89d569 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/HttpUtils.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/HttpUtils.java @@ -17,35 +17,45 @@ package com.github.sonus21.rqueue.utils; import com.github.sonus21.rqueue.config.RqueueConfig; +import com.github.sonus21.rqueue.serdes.SerializationUtils; import java.net.InetSocketAddress; -import java.net.Proxy; +import java.net.ProxySelector; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; import lombok.extern.slf4j.Slf4j; -import org.springframework.http.client.SimpleClientHttpRequestFactory; -import org.springframework.web.client.RestTemplate; @Slf4j public final class HttpUtils { private HttpUtils() {} - private static SimpleClientHttpRequestFactory getRequestFactory(RqueueConfig rqueueConfig) { - SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory(); - requestFactory.setReadTimeout(2 * Constants.ONE_MILLI_INT); - requestFactory.setConnectTimeout(2 * Constants.ONE_MILLI_INT); - if (StringUtils.isEmpty(rqueueConfig.getProxyHost())) { - return requestFactory; + private static HttpClient buildClient(RqueueConfig rqueueConfig) { + HttpClient.Builder builder = + HttpClient.newBuilder().connectTimeout(Duration.ofMillis(2 * Constants.ONE_MILLI_INT)); + if (!StringUtils.isEmpty(rqueueConfig.getProxyHost())) { + builder.proxy(ProxySelector.of( + new InetSocketAddress(rqueueConfig.getProxyHost(), rqueueConfig.getProxyPort()))); } - Proxy proxy = new Proxy( - rqueueConfig.getProxyType(), - new InetSocketAddress(rqueueConfig.getProxyHost(), rqueueConfig.getProxyPort())); - requestFactory.setProxy(proxy); - return requestFactory; + return builder.build(); } public static T readUrl(RqueueConfig rqueueConfig, String url, Class clazz) { try { - RestTemplate restTemplate = new RestTemplate(getRequestFactory(rqueueConfig)); - return restTemplate.getForObject(url, clazz); + HttpClient client = buildClient(rqueueConfig); + HttpRequest request = HttpRequest.newBuilder(URI.create(url)) + .timeout(Duration.ofMillis(2 * Constants.ONE_MILLI_INT)) + .header("Accept", "application/json") + .GET() + .build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() / 100 != 2) { + log.error("GET {} returned status {}", url, response.statusCode()); + return null; + } + return SerializationUtils.getSerDes().deserialize(response.body(), clazz); } catch (Exception e) { log.error("GET call failed for {}", url, e); return null; diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/condition/MissingRqueueSerDes.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/condition/MissingRqueueSerDes.java new file mode 100644 index 000000000..d1026e713 --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/condition/MissingRqueueSerDes.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ + +package com.github.sonus21.rqueue.utils.condition; + +import com.github.sonus21.rqueue.serdes.RqueueSerDes; +import org.springframework.beans.factory.ListableBeanFactory; +import org.springframework.context.annotation.Condition; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.core.type.AnnotatedTypeMetadata; + +public class MissingRqueueSerDes implements Condition { + + @Override + public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { + if (!(context.getBeanFactory() instanceof ListableBeanFactory)) { + return true; + } + ListableBeanFactory beanFactory = (ListableBeanFactory) context.getBeanFactory(); + return beanFactory.getBeanNamesForType(RqueueSerDes.class, false, false).length == 0; + } +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/condition/MissingRqueueTypeFactory.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/condition/MissingRqueueTypeFactory.java new file mode 100644 index 000000000..4b52f7893 --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/condition/MissingRqueueTypeFactory.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ + +package com.github.sonus21.rqueue.utils.condition; + +import com.github.sonus21.rqueue.serdes.RqueueTypeFactory; +import org.springframework.beans.factory.ListableBeanFactory; +import org.springframework.context.annotation.Condition; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.core.type.AnnotatedTypeMetadata; + +public class MissingRqueueTypeFactory implements Condition { + + @Override + public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { + if (!(context.getBeanFactory() instanceof ListableBeanFactory)) { + return true; + } + ListableBeanFactory beanFactory = (ListableBeanFactory) context.getBeanFactory(); + return beanFactory.getBeanNamesForType(RqueueTypeFactory.class, false, false).length == 0; + } +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/worker/RqueueWorkerRegistryImpl.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/worker/RqueueWorkerRegistryImpl.java index 70a9de9b6..2c2bf7cf4 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/worker/RqueueWorkerRegistryImpl.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/worker/RqueueWorkerRegistryImpl.java @@ -16,7 +16,6 @@ package com.github.sonus21.rqueue.worker; -import com.github.sonus21.rqueue.common.RqueueRedisTemplate; import com.github.sonus21.rqueue.config.RqueueConfig; import com.github.sonus21.rqueue.core.EndpointRegistry; import com.github.sonus21.rqueue.listener.QueueDetail; @@ -24,9 +23,10 @@ import com.github.sonus21.rqueue.models.registry.RqueueWorkerInfo; import com.github.sonus21.rqueue.models.registry.RqueueWorkerPollerMetadata; import com.github.sonus21.rqueue.models.registry.RqueueWorkerPollerView; +import com.github.sonus21.rqueue.serdes.RqueueSerDes; +import com.github.sonus21.rqueue.serdes.SerializationUtils; import com.github.sonus21.rqueue.utils.DateTimeUtils; import com.github.sonus21.rqueue.utils.QueueThreadPool; -import com.github.sonus21.rqueue.utils.SerializationUtils; import com.github.sonus21.rqueue.utils.StringUtils; import java.lang.management.ManagementFactory; import java.net.InetAddress; @@ -43,15 +43,18 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.context.ApplicationListener; import org.springframework.util.CollectionUtils; -import tools.jackson.databind.ObjectMapper; +/** + * Backend-agnostic worker registry. All heartbeat scheduling, in-memory bookkeeping, and view + * assembly lives here; storage is delegated to a {@link WorkerRegistryStore}, of which Redis + * and NATS JetStream KV provide concrete implementations. + */ @Slf4j public class RqueueWorkerRegistryImpl implements RqueueWorkerRegistry, ApplicationListener { - private final ObjectMapper objectMapper = SerializationUtils.createObjectMapper(); + private final RqueueSerDes serDes = SerializationUtils.getSerDes(); private final RqueueConfig rqueueConfig; - private final RqueueRedisTemplate workerTemplate; - private final RqueueRedisTemplate stringTemplate; + private final WorkerRegistryStore store; private final String workerId; private final String host; private final String pid; @@ -64,10 +67,9 @@ public class RqueueWorkerRegistryImpl private final Map capacityExhaustedCountByQueue = new ConcurrentHashMap<>(); private volatile long lastWorkerHeartbeatAt; - public RqueueWorkerRegistryImpl(RqueueConfig rqueueConfig) { + public RqueueWorkerRegistryImpl(RqueueConfig rqueueConfig, WorkerRegistryStore store) { this.rqueueConfig = rqueueConfig; - this.workerTemplate = new RqueueRedisTemplate<>(rqueueConfig.getConnectionFactory()); - this.stringTemplate = new RqueueRedisTemplate<>(rqueueConfig.getConnectionFactory()); + this.store = store; this.workerId = RqueueConfig.getBrokerId(); this.host = getHostName(); this.pid = getPid(); @@ -80,27 +82,22 @@ public void recordQueuePoll( if (!rqueueConfig.isWorkerRegistryEnabled()) { return; } - String registryQueueName = registryQueueName(queueDetail); + String trackingKey = consumerTrackingKey(queueDetail); long now = System.currentTimeMillis(); if (messageReceived) { - lastMessageAtByQueue.put(registryQueueName, now); + lastMessageAtByQueue.put(trackingKey, now); } - lastPollAtByQueue.put(registryQueueName, now); + lastPollAtByQueue.put(trackingKey, now); refreshWorkerInfoIfRequired(now); - if (!queueHeartbeatRequired(registryQueueName, now)) { + if (!queueHeartbeatRequired(trackingKey, now)) { return; } - RqueueWorkerPollerMetadata metadata = buildMetadata(registryQueueName, queueThreadPool); - try { - stringTemplate.putHashValue( - rqueueConfig.getWorkerRegistryQueueKey(registryQueueName), - workerId, - objectMapper.writeValueAsString(metadata)); - refreshQueueTtlIfRequired(registryQueueName, now); - lastQueueHeartbeatAt.put(registryQueueName, now); - } catch (Exception e) { - log.warn("Worker registry serialization failed for queue {}", registryQueueName, e); - } + publishHeartbeat( + registryQueueName(queueDetail), + trackingKey, + queueThreadPool, + now, + queueDetail.resolvedConsumerName()); } @Override @@ -109,11 +106,11 @@ public void recordQueueCapacityExhausted( if (!rqueueConfig.isWorkerRegistryEnabled()) { return; } - String registryQueueName = registryQueueName(queueDetail); + String trackingKey = consumerTrackingKey(queueDetail); long now = System.currentTimeMillis(); refreshWorkerInfoIfRequired(now); - lastCapacityExhaustedAtByQueue.put(registryQueueName, now); - capacityExhaustedCountByQueue.compute(registryQueueName, (key, count) -> { + lastCapacityExhaustedAtByQueue.put(trackingKey, now); + capacityExhaustedCountByQueue.compute(trackingKey, (key, count) -> { if (count == null) { return 1L; } @@ -122,19 +119,32 @@ public void recordQueueCapacityExhausted( } return count + 1L; }); - if (!queueHeartbeatRequired(registryQueueName, now)) { + if (!queueHeartbeatRequired(trackingKey, now)) { return; } - RqueueWorkerPollerMetadata metadata = buildMetadata(registryQueueName, queueThreadPool); + publishHeartbeat( + registryQueueName(queueDetail), + trackingKey, + queueThreadPool, + now, + queueDetail.resolvedConsumerName()); + } + + private void publishHeartbeat( + String registryQueueName, + String trackingKey, + QueueThreadPool queueThreadPool, + long now, + String consumerName) { + RqueueWorkerPollerMetadata metadata = buildMetadata(trackingKey, queueThreadPool, consumerName); try { - stringTemplate.putHashValue( - rqueueConfig.getWorkerRegistryQueueKey(registryQueueName), - workerId, - objectMapper.writeValueAsString(metadata)); - refreshQueueTtlIfRequired(registryQueueName, now); - lastQueueHeartbeatAt.put(registryQueueName, now); + String queueKey = rqueueConfig.getWorkerRegistryQueueKey(registryQueueName); + store.putQueueHeartbeat( + queueKey, heartbeatSubKey(consumerName), serDes.serializeAsString(metadata)); + refreshQueueTtlIfRequired(registryQueueName, trackingKey, now); + lastQueueHeartbeatAt.put(trackingKey, now); } catch (Exception e) { - log.warn("Worker registry serialization failed for queue {}", registryQueueName, e); + log.warn("Worker registry serialization failed for queue {}", trackingKey, e); } } @@ -144,46 +154,53 @@ public List getQueueWorkers(String queueName) { return Collections.emptyList(); } String queueKey = rqueueConfig.getWorkerRegistryQueueKey(queueName); - Map rawEntries = stringTemplate.getHashEntries(queueKey); + Map rawEntries = store.getQueueHeartbeats(queueKey); if (CollectionUtils.isEmpty(rawEntries)) { return Collections.emptyList(); } long now = System.currentTimeMillis(); long staleAfter = 2 * rqueueConfig.getWorkerRegistryQueueHeartbeatInterval().toMillis(); - Map metadataByWorkerId = new LinkedHashMap<>(); + // Key = KV sub-key (may be "workerId__consumerName"); value = deserialized metadata. + Map metadataBySubKey = new LinkedHashMap<>(); List toDelete = new ArrayList<>(); for (Map.Entry entry : rawEntries.entrySet()) { try { RqueueWorkerPollerMetadata metadata = - objectMapper.readValue(entry.getValue(), RqueueWorkerPollerMetadata.class); + serDes.deserialize(entry.getValue(), RqueueWorkerPollerMetadata.class); if (metadata == null || metadata.getWorkerId() == null) { toDelete.add(entry.getKey()); continue; } - // Lazy cleanup for entries that are far older than the queue hash retention window. + // Lazy cleanup for entries that are far older than the queue retention window. if (now - metadata.getLastPollAt() > rqueueConfig.getWorkerRegistryQueueTtl().toMillis()) { toDelete.add(entry.getKey()); continue; } - metadataByWorkerId.put(entry.getKey(), metadata); + metadataBySubKey.put(entry.getKey(), metadata); } catch (Exception e) { log.warn("Worker registry deserialization failed for queue {}", queueName, e); toDelete.add(entry.getKey()); } } if (!toDelete.isEmpty()) { - stringTemplate.deleteHashValues(queueKey, toDelete.toArray(new String[0])); + store.deleteQueueHeartbeats(queueKey, toDelete.toArray(new String[0])); } - if (metadataByWorkerId.isEmpty()) { + if (metadataBySubKey.isEmpty()) { return Collections.emptyList(); } - Map workerInfoById = getWorkerInfo(metadataByWorkerId.keySet()); + // Collect unique bare workerIds from the metadata bodies (not from KV sub-keys, which may + // be composite "workerId__consumerName" when multiple consumers share a queue). + java.util.Set bareWorkerIds = new java.util.LinkedHashSet<>(); + for (RqueueWorkerPollerMetadata m : metadataBySubKey.values()) { + bareWorkerIds.add(m.getWorkerId()); + } + Map workerInfoById = loadWorkerInfo(bareWorkerIds); List rows = new ArrayList<>(); - for (Map.Entry entry : metadataByWorkerId.entrySet()) { - String workerId = entry.getKey(); + for (Map.Entry entry : metadataBySubKey.entrySet()) { RqueueWorkerPollerMetadata metadata = entry.getValue(); - RqueueWorkerInfo workerInfo = workerInfoById.get(workerId); + String bareWorkerId = metadata.getWorkerId(); + RqueueWorkerInfo workerInfo = workerInfoById.get(bareWorkerId); long lastActivityAt = Math.max( metadata.getLastPollAt(), metadata.getLastCapacityExhaustedAt() == null @@ -192,7 +209,8 @@ public List getQueueWorkers(String queueName) { boolean stale = now - lastActivityAt > staleAfter || workerInfo == null; rows.add(RqueueWorkerPollerView.builder() .queue(queueName) - .workerId(workerId) + .workerId(bareWorkerId) + .consumerName(metadata.getConsumerName()) .host(workerInfo == null ? "unknown" : workerInfo.getHost()) .pid(workerInfo == null ? "" : workerInfo.getPid()) .status(stale ? "STALE" : "ACTIVE") @@ -239,7 +257,7 @@ private void refreshWorkerInfo(long now) { .startedAt(startedAt) .lastSeenAt(now) .build(); - workerTemplate.set( + store.putWorkerInfo( rqueueConfig.getWorkerRegistryKey(workerId), workerInfo, rqueueConfig.getWorkerRegistryWorkerTtl()); @@ -255,22 +273,24 @@ private boolean queueHeartbeatRequired(String queueName, long now) { >= rqueueConfig.getWorkerRegistryQueueHeartbeatInterval().toMillis(); } - private void refreshQueueTtlIfRequired(String queueName, long now) { + private void refreshQueueTtlIfRequired(String registryQueueName, String trackingKey, long now) { Duration ttl = rqueueConfig.getWorkerRegistryQueueTtl(); long refreshIntervalInMillis = Math.max(1000L, ttl.toMillis() / 2); - Long lastRefreshAt = lastQueueTtlRefreshAt.get(queueName); + Long lastRefreshAt = lastQueueTtlRefreshAt.get(trackingKey); if (lastRefreshAt != null && now - lastRefreshAt < refreshIntervalInMillis) { return; } - stringTemplate.expire(rqueueConfig.getWorkerRegistryQueueKey(queueName), ttl); - lastQueueTtlRefreshAt.put(queueName, now); + store.refreshQueueTtl(rqueueConfig.getWorkerRegistryQueueKey(registryQueueName), ttl); + lastQueueTtlRefreshAt.put(trackingKey, now); } private void cleanup() { - workerTemplate.delete(rqueueConfig.getWorkerRegistryKey(workerId)); + store.deleteWorkerInfo(rqueueConfig.getWorkerRegistryKey(workerId)); for (QueueDetail queueDetail : EndpointRegistry.getActiveQueueDetails()) { - stringTemplate.deleteHashValues( - rqueueConfig.getWorkerRegistryQueueKey(registryQueueName(queueDetail)), workerId); + String consumerName = queueDetail.resolvedConsumerName(); + store.deleteQueueHeartbeats( + rqueueConfig.getWorkerRegistryQueueKey(registryQueueName(queueDetail)), + heartbeatSubKey(consumerName)); } lastMessageAtByQueue.clear(); lastPollAtByQueue.clear(); @@ -282,9 +302,10 @@ private void cleanup() { } private RqueueWorkerPollerMetadata buildMetadata( - String registryQueueName, QueueThreadPool queueThreadPool) { + String registryQueueName, QueueThreadPool queueThreadPool, String consumerName) { return RqueueWorkerPollerMetadata.builder() .workerId(workerId) + .consumerName(consumerName) .lastPollAt(lastPollAtByQueue.getOrDefault(registryQueueName, 0L)) .lastMessageAt(lastMessageAtByQueue.get(registryQueueName)) .lastCapacityExhaustedAt(lastCapacityExhaustedAtByQueue.get(registryQueueName)) @@ -292,22 +313,13 @@ private RqueueWorkerPollerMetadata buildMetadata( .build(); } - private Map getWorkerInfo(Collection workerIds) { + private Map loadWorkerInfo(Collection workerIds) { List keys = new ArrayList<>(workerIds.size()); for (String workerId : workerIds) { keys.add(rqueueConfig.getWorkerRegistryKey(workerId)); } - List workerInfos = workerTemplate.mget(keys); - if (CollectionUtils.isEmpty(workerInfos)) { - return Collections.emptyMap(); - } - Map workerInfoById = new LinkedHashMap<>(); - for (RqueueWorkerInfo workerInfo : workerInfos) { - if (workerInfo != null && workerInfo.getWorkerId() != null) { - workerInfoById.put(workerInfo.getWorkerId(), workerInfo); - } - } - return workerInfoById; + Map result = store.getWorkerInfos(keys); + return result == null ? Collections.emptyMap() : result; } private static String formatAge(long now, Long time) { @@ -317,6 +329,24 @@ private static String formatAge(long now, Long time) { return DateTimeUtils.milliToHumanRepresentation(now - time); } + /** + * Returns the KV sub-key used to store this worker's heartbeat for a queue. When two + * consumers share the same queue name (e.g. two {@code @RqueueListener} methods on the same + * queue with different {@code consumerName}s), the consumer name is appended so each consumer + * gets its own independent heartbeat entry rather than overwriting the other's. + */ + private String heartbeatSubKey(String consumerName) { + if (consumerName == null || consumerName.isEmpty()) { + return workerId; + } + return workerId + "__" + consumerName; + } + + /** + * Returns the queue name used as the KV bucket prefix for heartbeats. Always the bare queue + * name (no consumer suffix) so that {@link #getQueueWorkers(String)} can find all consumers + * of a queue under a single prefix scan. + */ private static String registryQueueName(QueueDetail queueDetail) { if (queueDetail.isSystemGenerated() && !StringUtils.isEmpty(queueDetail.getPriorityGroup())) { return queueDetail.getPriorityGroup(); @@ -324,6 +354,19 @@ private static String registryQueueName(QueueDetail queueDetail) { return queueDetail.getName(); } + /** + * Returns the key used for all in-memory tracking maps (poll timestamps, heartbeat throttle, + * TTL refresh throttle, capacity exhaustion counts). When multiple {@code @RqueueListener} + * methods target the same queue with different consumer names, each consumer needs its own + * independent tracking entry — otherwise the first heartbeat stamps the shared key and + * suppresses all subsequent consumers during the throttle window. + */ + private static String consumerTrackingKey(QueueDetail queueDetail) { + String base = registryQueueName(queueDetail); + String cn = queueDetail.resolvedConsumerName(); + return (cn != null && !cn.isEmpty()) ? base + "#" + cn : base; + } + private static String getPid() { String runtimeName = ManagementFactory.getRuntimeMXBean().getName(); int index = runtimeName.indexOf('@'); diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/worker/WorkerRegistryStore.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/worker/WorkerRegistryStore.java new file mode 100644 index 000000000..af313d38d --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/worker/WorkerRegistryStore.java @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ + +package com.github.sonus21.rqueue.worker; + +import com.github.sonus21.rqueue.models.registry.RqueueWorkerInfo; +import java.time.Duration; +import java.util.Collection; +import java.util.Map; + +/** + * Backend-shaped persistence primitives for the worker registry. The single + * {@link RqueueWorkerRegistry} implementation in core orchestrates all heartbeat / view logic + * and delegates storage to one of these stores. Redis and NATS provide concrete impls. + * + *

Operations are intentionally narrow and KV-shaped so this abstraction maps cleanly onto + * both Redis (string keys + a hash per queue) and NATS JetStream KV (one bucket for worker + * infos, another bucket for queue heartbeats keyed by {@code "."}). + */ +public interface WorkerRegistryStore { + + /** + * Persist this worker's heartbeat record (host / pid / version / lastSeenAt). Writers SHOULD + * apply the supplied TTL; backends without per-key TTL (e.g. NATS KV without per-message TTL) + * may rely on a bucket-level {@code maxAge} configured to the same value. + */ + void putWorkerInfo(String workerKey, RqueueWorkerInfo info, Duration ttl); + + /** Best-effort delete; called on graceful shutdown. */ + void deleteWorkerInfo(String workerKey); + + /** + * Bulk-load worker infos by full key. Keys with no entry (or that fail to deserialize) are + * omitted from the returned map. The map is keyed by {@link RqueueWorkerInfo} worker ID + * ({@code getWorkerId()}). + */ + Map getWorkerInfos(Collection workerKeys); + + /** + * Write/overwrite this worker's heartbeat metadata for the given queue. Each (queueKey, + * workerId) pair is independent — concurrent writes from different workers must not collide. + */ + void putQueueHeartbeat(String queueKey, String workerId, String metadataJson); + + /** + * Read all worker heartbeats currently registered for this queue, keyed by workerId. Empty + * map if none. Implementations should not throw on a missing queue. + */ + Map getQueueHeartbeats(String queueKey); + + /** Remove specific worker heartbeats from a queue (lazy cleanup of stale entries). */ + void deleteQueueHeartbeats(String queueKey, String... workerIds); + + /** + * Refresh the queue-heartbeat container's TTL. On Redis this is {@code EXPIRE} on the hash + * key; on NATS KV the container TTL is the bucket's {@code maxAge} and individual writes + * implicitly reset entry expiry, so impls may treat this as a no-op. + */ + void refreshQueueTtl(String queueKey, Duration ttl); +} diff --git a/rqueue-core/src/main/resources/META-INF/additional-spring-configuration-metadata.json b/rqueue-core/src/main/resources/META-INF/additional-spring-configuration-metadata.json new file mode 100644 index 000000000..3ea19eae2 --- /dev/null +++ b/rqueue-core/src/main/resources/META-INF/additional-spring-configuration-metadata.json @@ -0,0 +1,251 @@ +{ + "groups": [], + "properties": [ + { + "name": "rqueue.nats.auto-create-streams", + "type": "java.lang.Boolean", + "description": "When true (default), Rqueue creates JetStream streams for each registered queue at startup via NatsStreamValidator. Set to false for locked-down JetStream accounts where the application credentials lack add_stream permission; NatsStreamValidator will then verify that every required stream already exists and abort boot with a clear error listing any missing ones.", + "defaultValue": true, + "sourceType": "com.github.sonus21.rqueue.spring.boot.RqueueNatsProperties" + }, + { + "name": "rqueue.nats.auto-create-consumers", + "type": "java.lang.Boolean", + "description": "When true (default), Rqueue lazily creates durable pull consumers on the first pop call for each (stream, consumerName) pair and caches the subscription in-process. Set to false to fail-fast on missing consumers instead.", + "defaultValue": true, + "sourceType": "com.github.sonus21.rqueue.spring.boot.RqueueNatsProperties" + }, + { + "name": "rqueue.nats.auto-create-dlq-stream", + "type": "java.lang.Boolean", + "description": "When true, Rqueue automatically creates a dead-letter queue stream for each registered queue that declares a deadLetterQueue in its @RqueueListener. Default is false.", + "defaultValue": false, + "sourceType": "com.github.sonus21.rqueue.spring.boot.RqueueNatsProperties" + }, + { + "name": "rqueue.nats.auto-create-kv-buckets", + "type": "java.lang.Boolean", + "description": "When true (default), each NATS-backed store lazily creates its KV bucket on first use. Set to false for locked-down JetStream accounts; NatsKvBucketValidator will verify at startup that every required bucket already exists and abort boot listing any missing ones. Pre-create the buckets with: nats kv add rqueue-queue-config / rqueue-jobs / rqueue-locks / rqueue-message-metadata / rqueue-workers / rqueue-worker-heartbeats.", + "defaultValue": true, + "sourceType": "com.github.sonus21.rqueue.spring.boot.RqueueNatsProperties" + }, + { + "name": "rqueue.nats.connection.url", + "type": "java.lang.String", + "description": "NATS server URL (e.g. nats://localhost:4222). Used when a single server URL is sufficient. Use rqueue.nats.connection.urls for a cluster.", + "sourceType": "com.github.sonus21.rqueue.spring.boot.RqueueNatsProperties$Connection" + }, + { + "name": "rqueue.nats.connection.credentials-path", + "type": "java.lang.String", + "description": "Path to a NATS credentials file (.creds) for NKey/JWT-based authentication.", + "sourceType": "com.github.sonus21.rqueue.spring.boot.RqueueNatsProperties$Connection" + }, + { + "name": "rqueue.nats.connection.username", + "type": "java.lang.String", + "description": "Username for NATS username/password authentication.", + "sourceType": "com.github.sonus21.rqueue.spring.boot.RqueueNatsProperties$Connection" + }, + { + "name": "rqueue.nats.connection.password", + "type": "java.lang.String", + "description": "Password for NATS username/password authentication.", + "sourceType": "com.github.sonus21.rqueue.spring.boot.RqueueNatsProperties$Connection" + }, + { + "name": "rqueue.nats.connection.token", + "type": "java.lang.String", + "description": "Token for NATS token-based authentication.", + "sourceType": "com.github.sonus21.rqueue.spring.boot.RqueueNatsProperties$Connection" + }, + { + "name": "rqueue.nats.connection.tls", + "type": "java.lang.Boolean", + "description": "Enable TLS for the NATS connection.", + "defaultValue": false, + "sourceType": "com.github.sonus21.rqueue.spring.boot.RqueueNatsProperties$Connection" + }, + { + "name": "rqueue.nats.connection.connection-name", + "type": "java.lang.String", + "description": "Logical name for this NATS connection, visible in server monitoring.", + "sourceType": "com.github.sonus21.rqueue.spring.boot.RqueueNatsProperties$Connection" + }, + { + "name": "rqueue.nats.connection.connect-timeout", + "type": "java.time.Duration", + "description": "Maximum time to wait for the initial connection to the NATS server.", + "sourceType": "com.github.sonus21.rqueue.spring.boot.RqueueNatsProperties$Connection" + }, + { + "name": "rqueue.nats.connection.reconnect-wait", + "type": "java.time.Duration", + "description": "Time to wait between reconnect attempts when the NATS connection is lost.", + "sourceType": "com.github.sonus21.rqueue.spring.boot.RqueueNatsProperties$Connection" + }, + { + "name": "rqueue.nats.connection.max-reconnects", + "type": "java.lang.Integer", + "description": "Maximum number of reconnect attempts. -1 means unlimited.", + "defaultValue": -1, + "sourceType": "com.github.sonus21.rqueue.spring.boot.RqueueNatsProperties$Connection" + }, + { + "name": "rqueue.nats.connection.ping-interval", + "type": "java.time.Duration", + "description": "Interval between server pings to detect a stale connection.", + "sourceType": "com.github.sonus21.rqueue.spring.boot.RqueueNatsProperties$Connection" + }, + { + "name": "rqueue.nats.stream.replicas", + "type": "java.lang.Integer", + "description": "Number of stream replicas. Must not exceed the number of JetStream-enabled servers in the cluster.", + "defaultValue": 1, + "sourceType": "com.github.sonus21.rqueue.spring.boot.RqueueNatsProperties$Stream" + }, + { + "name": "rqueue.nats.stream.storage", + "type": "java.lang.String", + "description": "Storage backend for streams: FILE (default, durable) or MEMORY (faster, lost on restart).", + "defaultValue": "FILE", + "sourceType": "com.github.sonus21.rqueue.spring.boot.RqueueNatsProperties$Stream" + }, + { + "name": "rqueue.nats.stream.retention", + "type": "java.lang.String", + "description": "Stream retention policy: LIMITS (default, bounded by max-age/max-bytes/max-messages), INTEREST (remove when no consumers), or WORK_QUEUE (remove on ack).", + "defaultValue": "LIMITS", + "sourceType": "com.github.sonus21.rqueue.spring.boot.RqueueNatsProperties$Stream" + }, + { + "name": "rqueue.nats.stream.max-age", + "type": "java.time.Duration", + "description": "Maximum age of messages in the stream before they are discarded. Default is 14 days.", + "defaultValue": "14d", + "sourceType": "com.github.sonus21.rqueue.spring.boot.RqueueNatsProperties$Stream" + }, + { + "name": "rqueue.nats.stream.max-bytes", + "type": "java.lang.Long", + "description": "Maximum total size in bytes of all messages in the stream. -1 means unlimited.", + "defaultValue": -1, + "sourceType": "com.github.sonus21.rqueue.spring.boot.RqueueNatsProperties$Stream" + }, + { + "name": "rqueue.nats.stream.max-messages", + "type": "java.lang.Long", + "description": "Maximum number of messages in the stream. -1 means unlimited.", + "defaultValue": -1, + "sourceType": "com.github.sonus21.rqueue.spring.boot.RqueueNatsProperties$Stream" + }, + { + "name": "rqueue.nats.stream.discard-policy", + "type": "java.lang.String", + "description": "Policy for discarding messages when stream limits are reached: OLD (default, discard oldest) or NEW (reject new messages).", + "defaultValue": "OLD", + "sourceType": "com.github.sonus21.rqueue.spring.boot.RqueueNatsProperties$Stream" + }, + { + "name": "rqueue.nats.stream.duplicate-window", + "type": "java.time.Duration", + "description": "Deduplication window for the Nats-Msg-Id header. Messages with the same ID published within this window are deduplicated by the server. Default is 2 minutes.", + "defaultValue": "2m", + "sourceType": "com.github.sonus21.rqueue.spring.boot.RqueueNatsProperties$Stream" + }, + { + "name": "rqueue.nats.consumer.ack-wait", + "type": "java.time.Duration", + "description": "Time the server waits for an ack before redelivering the message. Should be longer than the slowest expected message processing time. Default is 30 seconds.", + "defaultValue": "30s", + "sourceType": "com.github.sonus21.rqueue.spring.boot.RqueueNatsProperties$Consumer" + }, + { + "name": "rqueue.nats.consumer.max-deliver", + "type": "java.lang.Long", + "description": "Maximum number of delivery attempts for a message before it is sent to the DLQ (if configured). Default is 3.", + "defaultValue": 3, + "sourceType": "com.github.sonus21.rqueue.spring.boot.RqueueNatsProperties$Consumer" + }, + { + "name": "rqueue.nats.consumer.max-ack-pending", + "type": "java.lang.Long", + "description": "Maximum number of unacknowledged messages a consumer can hold. Default is 1000.", + "defaultValue": 1000, + "sourceType": "com.github.sonus21.rqueue.spring.boot.RqueueNatsProperties$Consumer" + }, + { + "name": "rqueue.nats.consumer.fetch-wait", + "type": "java.time.Duration", + "description": "Maximum time to wait for messages in a single fetch call before returning an empty result. Default is 2 seconds.", + "defaultValue": "2s", + "sourceType": "com.github.sonus21.rqueue.spring.boot.RqueueNatsProperties$Consumer" + }, + { + "name": "rqueue.nats.naming.stream-prefix", + "type": "java.lang.String", + "description": "Prefix applied to every JetStream stream name created by Rqueue (e.g. rqueue-js-my-queue). Default is 'rqueue-js-'.", + "defaultValue": "rqueue-js-", + "sourceType": "com.github.sonus21.rqueue.spring.boot.RqueueNatsProperties$Naming" + }, + { + "name": "rqueue.nats.naming.subject-prefix", + "type": "java.lang.String", + "description": "Prefix applied to every JetStream subject used by Rqueue (e.g. rqueue.js.my-queue). Default is 'rqueue.js.'.", + "defaultValue": "rqueue.js.", + "sourceType": "com.github.sonus21.rqueue.spring.boot.RqueueNatsProperties$Naming" + }, + { + "name": "rqueue.nats.naming.dlq-suffix", + "type": "java.lang.String", + "description": "Suffix appended to the stream and subject name for dead-letter queue streams (e.g. rqueue-js-my-queue-dlq). Default is '-dlq'.", + "defaultValue": "-dlq", + "sourceType": "com.github.sonus21.rqueue.spring.boot.RqueueNatsProperties$Naming" + } + ], + "hints": [ + { + "name": "rqueue.nats.stream.storage", + "values": [ + { + "value": "FILE", + "description": "Durable file-backed storage (default)." + }, + { + "value": "MEMORY", + "description": "In-memory storage; faster but lost on server restart." + } + ] + }, + { + "name": "rqueue.nats.stream.retention", + "values": [ + { + "value": "LIMITS", + "description": "Retain messages subject to max-age, max-bytes, and max-messages limits (default)." + }, + { + "value": "INTEREST", + "description": "Remove messages once all active consumers have acknowledged them." + }, + { + "value": "WORK_QUEUE", + "description": "Remove messages immediately after they are acknowledged by any consumer." + } + ] + }, + { + "name": "rqueue.nats.stream.discard-policy", + "values": [ + { + "value": "OLD", + "description": "Discard the oldest messages when limits are reached (default)." + }, + { + "value": "NEW", + "description": "Reject new publish calls when limits are reached." + } + ] + } + ] +} diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/converter/GenericMessageConverterTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/converter/GenericMessageConverterTest.java index b6c411f06..53b516908 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/converter/GenericMessageConverterTest.java +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/converter/GenericMessageConverterTest.java @@ -22,6 +22,7 @@ import com.github.sonus21.TestBase; import com.github.sonus21.rqueue.CoreUnitTest; import com.github.sonus21.rqueue.listener.RqueueMessageHeaders; +import com.github.sonus21.rqueue.serdes.SerializationUtils; import java.lang.reflect.GenericArrayType; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; @@ -47,6 +48,7 @@ import tools.jackson.core.JsonParser; import tools.jackson.databind.DeserializationContext; import tools.jackson.databind.JsonNode; +import tools.jackson.databind.ObjectMapper; import tools.jackson.databind.SerializationContext; import tools.jackson.databind.deser.std.StdDeserializer; import tools.jackson.databind.module.SimpleModule; @@ -165,6 +167,45 @@ void envelopeEventWithInheritedTypeToAndFromMessage() { assertEquals(event, fromMessage); } + /** + * Verifies that messages serialised with the old {@code String msg} wire format (before the field + * was changed to {@code byte[]}) are still readable by the current converter. Each case builds + * an old-format JSON envelope via {@link OldMsg} and feeds it directly to + * {@link GenericMessageConverter#fromMessage} to confirm the payload comes back intact. + */ + @Test + void backwardCompatibility() throws Exception { + ObjectMapper mapper = SerializationUtils.getObjectMapper(); + + Comment unicodeComment = new Comment("u-1", "Héllo 你好 🎉"); + Comment asciiComment = new Comment("a-1", "plain ascii"); + Email specialEmail = new Email("e-2", "subject: re: hello & goodbye"); + + // 5 cases: class name is straightforward for non-generic POJOs + Object[][] cases = { + {comment, comment.getClass().getName()}, + {email, email.getClass().getName()}, + {unicodeComment, unicodeComment.getClass().getName()}, + {asciiComment, asciiComment.getClass().getName()}, + {specialEmail, specialEmail.getClass().getName()}, + }; + + for (Object[] c : cases) { + Object payload = c[0]; + String className = (String) c[1]; + + // Build old-format JSON: {"msg":"","name":""} + String innerJson = mapper.writeValueAsString(payload); + String oldFormatJson = mapper.writeValueAsString(new OldMsg(innerJson, className)); + + Object restored = + genericMessageConverter.fromMessage(new GenericMessage<>(oldFormatJson), null); + + assertEquals( + payload, restored, "old String-format data must deserialise correctly for " + className); + } + } + @Test void genericEnvelopeToAndFromMessage() { GenericTestData data = new GenericTestData<>(10, comment); @@ -507,6 +548,15 @@ public int hashCode() { } } + /** Mirrors the old wire format of the private {@code Msg} class (before {@code msg} became {@code byte[]}). */ + @Data + @NoArgsConstructor + @AllArgsConstructor + private static class OldMsg { + private String msg; + private String name; + } + public static class ServiceLoadedPayloadModule extends SimpleModule { public ServiceLoadedPayloadModule() { diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/converter/JsonMessageConverterTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/converter/JsonMessageConverterTest.java index 711709063..e70df8f60 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/converter/JsonMessageConverterTest.java +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/converter/JsonMessageConverterTest.java @@ -22,6 +22,8 @@ import com.github.sonus21.TestBase; import com.github.sonus21.rqueue.CoreUnitTest; +import com.github.sonus21.rqueue.serdes.RqJacksonSerDes; +import com.github.sonus21.rqueue.serdes.RqueueSerDes; import java.util.UUID; import lombok.AllArgsConstructor; import lombok.Data; @@ -32,17 +34,16 @@ import org.springframework.messaging.Message; import org.springframework.messaging.converter.MessageConverter; import tools.jackson.databind.DeserializationFeature; -import tools.jackson.databind.ObjectMapper; import tools.jackson.databind.json.JsonMapper; @CoreUnitTest class JsonMessageConverterTest extends TestBase { - private final ObjectMapper objectMapper = JsonMapper.builder() + private final RqueueSerDes serDes = new RqJacksonSerDes(JsonMapper.builder() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) - .build(); + .build()); private final MessageConverter messageConverter = new JsonMessageConverter(); - private final MessageConverter messageConverter2 = new JsonMessageConverter(objectMapper); + private final MessageConverter messageConverter2 = new JsonMessageConverter(serDes); @Test void fromMessage() { diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/RqueueEndpointManagerTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/RqueueEndpointManagerTest.java index bd560e8b8..3c56ddfb2 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/RqueueEndpointManagerTest.java +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/RqueueEndpointManagerTest.java @@ -22,6 +22,7 @@ import com.github.sonus21.rqueue.CoreUnitTest; import com.github.sonus21.rqueue.config.RqueueConfig; import com.github.sonus21.rqueue.core.impl.RqueueEndpointManagerImpl; +import com.github.sonus21.rqueue.core.spi.redis.RedisMessageBroker; import org.apache.commons.lang3.reflect.FieldUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -44,7 +45,10 @@ class RqueueEndpointManagerTest extends TestBase { public void init() throws IllegalAccessException { MockitoAnnotations.openMocks(this); rqueueEndpointManager = new RqueueEndpointManagerImpl( - rqueueMessageTemplate, new DefaultRqueueMessageConverter(), null); + rqueueMessageTemplate, + new RedisMessageBroker(rqueueMessageTemplate), + new DefaultRqueueMessageConverter(), + null); FieldUtils.writeField(rqueueEndpointManager, "rqueueConfig", rqueueConfig, true); EndpointRegistry.delete(); diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/RqueueMessageEnqueuerTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/RqueueMessageEnqueuerTest.java index 80e5dfd43..0a0f45563 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/RqueueMessageEnqueuerTest.java +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/RqueueMessageEnqueuerTest.java @@ -27,9 +27,10 @@ import com.github.sonus21.rqueue.CoreUnitTest; import com.github.sonus21.rqueue.config.RqueueConfig; import com.github.sonus21.rqueue.core.impl.RqueueMessageEnqueuerImpl; +import com.github.sonus21.rqueue.core.spi.redis.RedisMessageBroker; import com.github.sonus21.rqueue.listener.QueueDetail; +import com.github.sonus21.rqueue.service.RqueueMessageMetadataService; import com.github.sonus21.rqueue.utils.TestUtils; -import com.github.sonus21.rqueue.web.service.RqueueMessageMetadataService; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mock; @@ -59,7 +60,10 @@ class RqueueMessageEnqueuerTest extends TestBase { public void init() throws IllegalAccessException { MockitoAnnotations.openMocks(this); rqueueMessageEnqueuer = new RqueueMessageEnqueuerImpl( - rqueueMessageTemplate, new DefaultRqueueMessageConverter(), null); + rqueueMessageTemplate, + new RedisMessageBroker(rqueueMessageTemplate), + new DefaultRqueueMessageConverter(), + null); EndpointRegistry.delete(); EndpointRegistry.register(queueDetail); writeField(rqueueMessageEnqueuer, "rqueueConfig", rqueueConfig, true); diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/RqueueMessageTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/RqueueMessageTest.java index 9340c5dd1..c123f75e0 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/RqueueMessageTest.java +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/RqueueMessageTest.java @@ -23,7 +23,7 @@ import com.github.sonus21.TestBase; import com.github.sonus21.rqueue.CoreUnitTest; -import com.github.sonus21.rqueue.utils.SerializationUtils; +import com.github.sonus21.rqueue.serdes.SerializationUtils; import java.util.UUID; import org.junit.jupiter.api.Test; import tools.jackson.core.JacksonException; @@ -31,8 +31,7 @@ @CoreUnitTest class RqueueMessageTest extends TestBase { - - private final ObjectMapper objectMapper = SerializationUtils.createObjectMapper(); + private final ObjectMapper objectMapper = SerializationUtils.getObjectMapper(); private final String queueName = "test-queue"; private final String queueMessage = "This is a test message"; private final Integer retryCount = 3; diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/impl/BaseMessageSenderMetadataTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/impl/BaseMessageSenderMetadataTest.java new file mode 100644 index 000000000..a532ef54c --- /dev/null +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/impl/BaseMessageSenderMetadataTest.java @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2020-2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ + +package com.github.sonus21.rqueue.core.impl; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.github.sonus21.TestBase; +import com.github.sonus21.rqueue.CoreUnitTest; +import com.github.sonus21.rqueue.config.RqueueConfig; +import com.github.sonus21.rqueue.core.DefaultRqueueMessageConverter; +import com.github.sonus21.rqueue.core.EndpointRegistry; +import com.github.sonus21.rqueue.core.RqueueMessageEnqueuer; +import com.github.sonus21.rqueue.core.RqueueMessageIdGenerator; +import com.github.sonus21.rqueue.core.RqueueMessageTemplate; +import com.github.sonus21.rqueue.core.spi.Capabilities; +import com.github.sonus21.rqueue.core.spi.MessageBroker; +import com.github.sonus21.rqueue.listener.QueueDetail; +import com.github.sonus21.rqueue.listener.RqueueMessageHeaders; +import com.github.sonus21.rqueue.service.RqueueMessageMetadataService; +import com.github.sonus21.rqueue.utils.TestUtils; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.converter.MessageConverter; + +@CoreUnitTest +class BaseMessageSenderMetadataTest extends TestBase { + + private static final String queue = "test-queue-metadata"; + private static final QueueDetail queueDetail = TestUtils.createQueueDetail(queue); + private static final RqueueMessageIdGenerator FIXED_MESSAGE_ID_GENERATOR = () -> "metadata-id"; + + private final MessageConverter messageConverter = new DefaultRqueueMessageConverter(); + private final MessageHeaders messageHeaders = RqueueMessageHeaders.emptyMessageHeaders(); + + @Mock + private RqueueMessageMetadataService rqueueMessageMetadataService; + + @Mock + private RqueueMessageTemplate messageTemplate; + + @Mock + private MessageBroker messageBroker; + + private RqueueConfig rqueueConfig; + private RqueueMessageEnqueuer enqueuer; + + @BeforeAll + public static void init0() { + EndpointRegistry.delete(); + EndpointRegistry.register(queueDetail); + } + + @AfterAll + public static void clean() { + EndpointRegistry.delete(); + } + + @BeforeEach + public void init() throws IllegalAccessException { + MockitoAnnotations.openMocks(this); + rqueueConfig = new RqueueConfig(null, null, true, 2); + rqueueConfig.setMessageDurabilityInMinute(10080); + enqueuer = new RqueueMessageEnqueuerImpl( + messageTemplate, + messageBroker, + messageConverter, + messageHeaders, + FIXED_MESSAGE_ID_GENERATOR); + FieldUtils.writeField(enqueuer, "rqueueConfig", rqueueConfig, true); + FieldUtils.writeField( + enqueuer, "rqueueMessageMetadataService", rqueueMessageMetadataService, true); + lenient().doNothing().when(rqueueMessageMetadataService).save(any(), any(), anyBoolean()); + } + + @Test + void redisCapabilitiesSaveMetadata() { + when(messageBroker.capabilities()).thenReturn(Capabilities.REDIS_DEFAULTS); + String id = enqueuer.enqueue(queue, "redis-payload"); + assertEquals("metadata-id", id); + verify(rqueueMessageMetadataService).save(any(), any(), anyBoolean()); + } + + @Test + void natsLikeCapabilitiesSkipMetadata() { + Capabilities caps = new Capabilities(true, false, false, false); + when(messageBroker.capabilities()).thenReturn(caps); + + String id = enqueuer.enqueue(queue, "nats-payload"); + assertEquals("metadata-id", id); + verify(rqueueMessageMetadataService, never()).save(any(), any(), anyBoolean()); + } +} diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/impl/ReactiveRqueueMessageEnqueuerBrokerRoutingTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/impl/ReactiveRqueueMessageEnqueuerBrokerRoutingTest.java new file mode 100644 index 000000000..4c776e275 --- /dev/null +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/impl/ReactiveRqueueMessageEnqueuerBrokerRoutingTest.java @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ + +package com.github.sonus21.rqueue.core.impl; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.github.sonus21.TestBase; +import com.github.sonus21.rqueue.CoreUnitTest; +import com.github.sonus21.rqueue.config.RqueueConfig; +import com.github.sonus21.rqueue.core.DefaultRqueueMessageConverter; +import com.github.sonus21.rqueue.core.EndpointRegistry; +import com.github.sonus21.rqueue.core.RqueueMessage; +import com.github.sonus21.rqueue.core.RqueueMessageIdGenerator; +import com.github.sonus21.rqueue.core.RqueueMessageTemplate; +import com.github.sonus21.rqueue.core.spi.Capabilities; +import com.github.sonus21.rqueue.core.spi.MessageBroker; +import com.github.sonus21.rqueue.listener.QueueDetail; +import com.github.sonus21.rqueue.listener.RqueueMessageHeaders; +import com.github.sonus21.rqueue.service.RqueueMessageMetadataService; +import com.github.sonus21.rqueue.utils.TestUtils; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.converter.MessageConverter; +import reactor.core.publisher.Mono; +import reactor.test.StepVerifier; + +@CoreUnitTest +class ReactiveRqueueMessageEnqueuerBrokerRoutingTest extends TestBase { + + private static final String queue = "broker-routing-queue"; + private static final QueueDetail queueDetail = TestUtils.createQueueDetail(queue); + private static final RqueueMessageIdGenerator FIXED_ID = () -> "fixed-id"; + + private final MessageConverter messageConverter = new DefaultRqueueMessageConverter(); + private final MessageHeaders messageHeaders = RqueueMessageHeaders.emptyMessageHeaders(); + + @Mock + private RqueueMessageMetadataService rqueueMessageMetadataService; + + @Mock + private RqueueMessageTemplate messageTemplate; + + @Mock + private MessageBroker messageBroker; + + private ReactiveRqueueMessageEnqueuerImpl enqueuer; + + @BeforeAll + static void init0() { + EndpointRegistry.delete(); + EndpointRegistry.register(queueDetail); + } + + @AfterAll + static void clean() { + EndpointRegistry.delete(); + } + + @BeforeEach + void init() throws IllegalAccessException { + MockitoAnnotations.openMocks(this); + RqueueConfig rqueueConfig = new RqueueConfig(null, null, true, 2); + rqueueConfig.setMessageDurabilityInMinute(10080); + enqueuer = new ReactiveRqueueMessageEnqueuerImpl( + messageTemplate, messageBroker, messageConverter, messageHeaders, FIXED_ID); + FieldUtils.writeField(enqueuer, "rqueueConfig", rqueueConfig, true); + FieldUtils.writeField( + enqueuer, "rqueueMessageMetadataService", rqueueMessageMetadataService, true); + lenient() + .when(messageBroker.capabilities()) + .thenReturn(new Capabilities(true, false, false, true)); + lenient() + .when(rqueueMessageMetadataService.saveReactive(any(), any(), anyBoolean())) + .thenReturn(Mono.just(Boolean.TRUE)); + } + + @Test + void enqueueReactive_routesThroughBroker() { + when(messageBroker.enqueueReactive(any(QueueDetail.class), any(RqueueMessage.class))) + .thenReturn(Mono.empty()); + + StepVerifier.create(enqueuer.enqueue(queue, "payload")) + .expectNext("fixed-id") + .verifyComplete(); + + verify(messageBroker, times(1)) + .enqueueReactive(any(QueueDetail.class), any(RqueueMessage.class)); + verify(messageTemplate, never()).addReactiveMessage(eq(queue), any()); + } + + @Test + void enqueueInReactive_routesThroughBrokerDelayed() { + when(messageBroker.enqueueWithDelayReactive( + any(QueueDetail.class), any(RqueueMessage.class), eq(5_000L))) + .thenReturn(Mono.empty()); + + StepVerifier.create(enqueuer.enqueueIn(queue, "payload", 5_000L)) + .expectNext("fixed-id") + .verifyComplete(); + + verify(messageBroker, times(1)) + .enqueueWithDelayReactive(any(QueueDetail.class), any(RqueueMessage.class), eq(5_000L)); + verify(messageTemplate, never()).addReactiveMessageWithDelay(any(), any(), any()); + } +} diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/impl/RqueueEndpointManagerImplTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/impl/RqueueEndpointManagerImplTest.java index 6c3e1ccb9..11a9adc7c 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/impl/RqueueEndpointManagerImplTest.java +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/impl/RqueueEndpointManagerImplTest.java @@ -29,14 +29,16 @@ import com.github.sonus21.rqueue.core.EndpointRegistry; import com.github.sonus21.rqueue.core.RqueueEndpointManager; import com.github.sonus21.rqueue.core.RqueueMessageTemplate; +import com.github.sonus21.rqueue.core.spi.MessageBroker; +import com.github.sonus21.rqueue.core.spi.redis.RedisMessageBroker; import com.github.sonus21.rqueue.dao.RqueueSystemConfigDao; import com.github.sonus21.rqueue.listener.RqueueMessageHeaders; import com.github.sonus21.rqueue.models.db.QueueConfig; import com.github.sonus21.rqueue.models.request.PauseUnpauseQueueRequest; import com.github.sonus21.rqueue.models.response.BaseResponse; +import com.github.sonus21.rqueue.service.RqueueUtilityService; import com.github.sonus21.rqueue.utils.PriorityUtils; import com.github.sonus21.rqueue.utils.TestUtils; -import com.github.sonus21.rqueue.web.service.RqueueUtilityService; import org.apache.commons.lang3.reflect.FieldUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -72,8 +74,8 @@ class RqueueEndpointManagerImplTest extends TestBase { @BeforeEach public void init() throws IllegalAccessException { MockitoAnnotations.openMocks(this); - rqueueEndpointManager = - new RqueueEndpointManagerImpl(messageTemplate, messageConverter, messageHeaders); + rqueueEndpointManager = new RqueueEndpointManagerImpl( + messageTemplate, new RedisMessageBroker(messageTemplate), messageConverter, messageHeaders); RqueueConfig rqueueConfig = new RqueueConfig(redisConnectionFactory, null, false, 1); FieldUtils.writeField(rqueueEndpointManager, "rqueueConfig", rqueueConfig, true); FieldUtils.writeField( @@ -185,4 +187,29 @@ void pauseQueueWithPriority() { .pauseUnpauseQueue(request); assertFalse(rqueueEndpointManager.pauseUnpauseQueue(queue, priorities[0], false)); } + + /** + * registerQueue must consult {@link MessageBroker#validateQueueName(String)}; an exception there + * must propagate so callers fail fast on illegal names (e.g. dots when running on NATS). + */ + @Test + void registerQueue_propagatesBrokerValidationFailure() throws IllegalAccessException { + MessageBroker rejectingBroker = new RedisMessageBroker(messageTemplate) { + @Override + public void validateQueueName(String queueName) { + if (queueName.contains(".")) { + throw new IllegalArgumentException("dot rejected: " + queueName); + } + } + }; + RqueueEndpointManager mgr = new RqueueEndpointManagerImpl( + messageTemplate, rejectingBroker, messageConverter, messageHeaders); + RqueueConfig rqueueConfig = new RqueueConfig(redisConnectionFactory, null, false, 1); + FieldUtils.writeField(mgr, "rqueueConfig", rqueueConfig, true); + + IllegalArgumentException ex = + assertThrows(IllegalArgumentException.class, () -> mgr.registerQueue("orders.us")); + assertTrue(ex.getMessage().contains("orders.us")); + assertFalse(mgr.isQueueRegistered("orders.us")); + } } diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/impl/RqueueMessageEnqueuerBrokerRoutingTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/impl/RqueueMessageEnqueuerBrokerRoutingTest.java new file mode 100644 index 000000000..4a5f56a37 --- /dev/null +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/impl/RqueueMessageEnqueuerBrokerRoutingTest.java @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ + +package com.github.sonus21.rqueue.core.impl; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.github.sonus21.TestBase; +import com.github.sonus21.rqueue.CoreUnitTest; +import com.github.sonus21.rqueue.config.RqueueConfig; +import com.github.sonus21.rqueue.core.DefaultRqueueMessageConverter; +import com.github.sonus21.rqueue.core.EndpointRegistry; +import com.github.sonus21.rqueue.core.RqueueMessage; +import com.github.sonus21.rqueue.core.RqueueMessageEnqueuer; +import com.github.sonus21.rqueue.core.RqueueMessageIdGenerator; +import com.github.sonus21.rqueue.core.RqueueMessageTemplate; +import com.github.sonus21.rqueue.core.spi.Capabilities; +import com.github.sonus21.rqueue.core.spi.MessageBroker; +import com.github.sonus21.rqueue.core.spi.redis.RedisMessageBroker; +import com.github.sonus21.rqueue.listener.QueueDetail; +import com.github.sonus21.rqueue.listener.RqueueMessageHeaders; +import com.github.sonus21.rqueue.service.RqueueMessageMetadataService; +import com.github.sonus21.rqueue.utils.TestUtils; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.converter.MessageConverter; + +/** + * Pins the non-reactive enqueue path. Verifies that {@code BaseMessageSender.enqueue()} always + * routes through the {@link MessageBroker} regardless of backend, and never falls through to a + * Redis-shaped {@code messageTemplate.addMessage()} call. Regression coverage for the NPE that + * surfaced when a NATS-backed enqueuer was constructed against a template with a {@code null} + * {@code RedisTemplate}. + */ +@CoreUnitTest +class RqueueMessageEnqueuerBrokerRoutingTest extends TestBase { + + private static final String queue = "broker-routing-queue-sync"; + private static final QueueDetail queueDetail = TestUtils.createQueueDetail(queue); + private static final RqueueMessageIdGenerator FIXED_ID = () -> "fixed-id"; + // NATS-shaped: not the primary-handler-dispatch (Redis) capability set. + private static final Capabilities NATS_LIKE = new Capabilities(true, false, false, false); + + private final MessageConverter messageConverter = new DefaultRqueueMessageConverter(); + private final MessageHeaders messageHeaders = RqueueMessageHeaders.emptyMessageHeaders(); + + @Mock + private RqueueMessageMetadataService rqueueMessageMetadataService; + + @Mock + private RqueueMessageTemplate messageTemplate; + + @Mock + private MessageBroker messageBroker; + + @BeforeAll + static void init0() { + EndpointRegistry.delete(); + EndpointRegistry.register(queueDetail); + } + + @AfterAll + static void clean() { + EndpointRegistry.delete(); + } + + @BeforeEach + void init() { + MockitoAnnotations.openMocks(this); + lenient().doNothing().when(rqueueMessageMetadataService).save(any(), any(), anyBoolean()); + } + + private RqueueMessageEnqueuer newEnqueuer(MessageBroker broker) throws IllegalAccessException { + RqueueConfig rqueueConfig = new RqueueConfig(null, null, true, 2); + rqueueConfig.setMessageDurabilityInMinute(10080); + RqueueMessageEnqueuer enqueuer = new RqueueMessageEnqueuerImpl( + messageTemplate, broker, messageConverter, messageHeaders, FIXED_ID); + FieldUtils.writeField(enqueuer, "rqueueConfig", rqueueConfig, true); + FieldUtils.writeField( + enqueuer, "rqueueMessageMetadataService", rqueueMessageMetadataService, true); + return enqueuer; + } + + @Test + void enqueue_routesThroughBroker_natsBackend() throws IllegalAccessException { + when(messageBroker.capabilities()).thenReturn(NATS_LIKE); + RqueueMessageEnqueuer enqueuer = newEnqueuer(messageBroker); + + String id = enqueuer.enqueue(queue, "payload"); + + assertEquals("fixed-id", id); + verify(messageBroker, times(1)) + .enqueue(any(QueueDetail.class), isNull(), any(RqueueMessage.class)); + // The Redis-shaped template path must never be touched — that was the original NPE source. + verify(messageTemplate, never()).addMessage(any(), any()); + verify(messageTemplate, never()).addMessageWithDelay(any(), any(), any()); + // NATS capabilities advertise !usesPrimaryHandlerDispatch — metadata save is skipped. + verify(rqueueMessageMetadataService, never()).save(any(), any(), anyBoolean()); + } + + @Test + void enqueueIn_routesThroughBrokerDelayed_natsBackend() throws IllegalAccessException { + when(messageBroker.capabilities()).thenReturn(NATS_LIKE); + RqueueMessageEnqueuer enqueuer = newEnqueuer(messageBroker); + + String id = enqueuer.enqueueIn(queue, "payload", 5_000L); + + assertEquals("fixed-id", id); + verify(messageBroker, times(1)) + .enqueueWithDelay(any(QueueDetail.class), any(RqueueMessage.class), eq(5_000L)); + verify(messageTemplate, never()).addMessageWithDelay(any(), any(), any()); + } + + @Test + void enqueue_routesThroughRedisBroker_redisBackend() throws IllegalAccessException { + // Redis path uses a real RedisMessageBroker that delegates to template.addMessage. No + // direct messageTemplate calls from BaseMessageSender — they all go through the broker. + RedisMessageBroker redisBroker = new RedisMessageBroker(messageTemplate); + RqueueMessageEnqueuer enqueuer = newEnqueuer(redisBroker); + + String id = enqueuer.enqueue(queue, "payload"); + + assertEquals("fixed-id", id); + verify(messageTemplate, times(1)) + .addMessage(eq(queueDetail.getQueueName()), any(RqueueMessage.class)); + verify(messageTemplate, never()).addMessageWithDelay(any(), any(), any()); + // Redis capabilities have usesPrimaryHandlerDispatch=true — metadata is saved. + verify(rqueueMessageMetadataService).save(any(), any(), anyBoolean()); + } +} diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/impl/RqueueMessageEnqueuerImplTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/impl/RqueueMessageEnqueuerImplTest.java index 0c817570d..c1fb9a82c 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/impl/RqueueMessageEnqueuerImplTest.java +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/impl/RqueueMessageEnqueuerImplTest.java @@ -31,12 +31,13 @@ import com.github.sonus21.rqueue.core.RqueueMessageEnqueuer; import com.github.sonus21.rqueue.core.RqueueMessageIdGenerator; import com.github.sonus21.rqueue.core.RqueueMessageTemplate; +import com.github.sonus21.rqueue.core.spi.redis.RedisMessageBroker; import com.github.sonus21.rqueue.listener.QueueDetail; import com.github.sonus21.rqueue.listener.RqueueMessageHeaders; import com.github.sonus21.rqueue.models.db.MessageMetadata; +import com.github.sonus21.rqueue.service.RqueueMessageMetadataService; import com.github.sonus21.rqueue.utils.Constants; import com.github.sonus21.rqueue.utils.TestUtils; -import com.github.sonus21.rqueue.web.service.RqueueMessageMetadataService; import java.util.UUID; import org.apache.commons.lang3.reflect.FieldUtils; import org.junit.jupiter.api.AfterAll; @@ -85,7 +86,11 @@ public void init() throws IllegalAccessException { rqueueConfig = new RqueueConfig(null, null, true, 2); rqueueConfig.setMessageDurabilityInMinute(10080); rqueueMessageEnqueuer = new RqueueMessageEnqueuerImpl( - messageTemplate, messageConverter, messageHeaders, FIXED_MESSAGE_ID_GENERATOR); + messageTemplate, + new RedisMessageBroker(messageTemplate), + messageConverter, + messageHeaders, + FIXED_MESSAGE_ID_GENERATOR); FieldUtils.writeField(rqueueMessageEnqueuer, "rqueueConfig", rqueueConfig, true); FieldUtils.writeField( rqueueMessageEnqueuer, "rqueueMessageMetadataService", rqueueMessageMetadataService, true); diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/impl/RqueueMessageManagerImplTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/impl/RqueueMessageManagerImplTest.java index debf9a685..c037e55c2 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/impl/RqueueMessageManagerImplTest.java +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/impl/RqueueMessageManagerImplTest.java @@ -33,20 +33,20 @@ import com.github.sonus21.TestBase; import com.github.sonus21.rqueue.CoreUnitTest; -import com.github.sonus21.rqueue.common.RqueueLockManager; import com.github.sonus21.rqueue.config.RqueueConfig; import com.github.sonus21.rqueue.core.DefaultRqueueMessageConverter; import com.github.sonus21.rqueue.core.EndpointRegistry; import com.github.sonus21.rqueue.core.RqueueMessage; import com.github.sonus21.rqueue.core.RqueueMessageManager; import com.github.sonus21.rqueue.core.RqueueMessageTemplate; +import com.github.sonus21.rqueue.core.spi.redis.RedisMessageBroker; import com.github.sonus21.rqueue.listener.QueueDetail; import com.github.sonus21.rqueue.models.MessageMoveResult; import com.github.sonus21.rqueue.models.db.MessageMetadata; +import com.github.sonus21.rqueue.service.RqueueMessageMetadataService; import com.github.sonus21.rqueue.utils.MessageMetadataTestUtils; import com.github.sonus21.rqueue.utils.PriorityUtils; import com.github.sonus21.rqueue.utils.TestUtils; -import com.github.sonus21.rqueue.web.service.RqueueMessageMetadataService; import java.time.Duration; import java.util.Collections; import java.util.List; @@ -84,9 +84,6 @@ class RqueueMessageManagerImplTest extends TestBase { messageConverter, queueNameWithPriority, message); private final RqueueMessage rqueueMessage2 = messageMetadata2.getRqueueMessage(); - @Mock - private RqueueLockManager rqueueLockManager; - @Mock private RqueueMessageMetadataService rqueueMessageMetadataService; @@ -104,13 +101,15 @@ class RqueueMessageManagerImplTest extends TestBase { @BeforeEach public void init() throws IllegalAccessException { MockitoAnnotations.openMocks(this); - rqueueMessageManager = - new RqueueMessageManagerImpl(rqueueMessageTemplate, messageConverter, null); + rqueueMessageManager = new RqueueMessageManagerImpl( + rqueueMessageTemplate, + new RedisMessageBroker(rqueueMessageTemplate), + messageConverter, + null); EndpointRegistry.delete(); EndpointRegistry.register(queueDetail); EndpointRegistry.register(queueDetail2); writeField(rqueueMessageManager, "rqueueConfig", rqueueConfig, true); - writeField(rqueueMessageManager, "rqueueLockManager", rqueueLockManager, true); writeField( rqueueMessageManager, "rqueueMessageMetadataService", rqueueMessageMetadataService, true); } diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/spi/redis/RedisMessageBrokerDelegationTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/spi/redis/RedisMessageBrokerDelegationTest.java new file mode 100644 index 000000000..794dc52af --- /dev/null +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/spi/redis/RedisMessageBrokerDelegationTest.java @@ -0,0 +1,244 @@ +/* + * Copyright (c) 2020-2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ + +package com.github.sonus21.rqueue.core.spi.redis; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import com.github.sonus21.TestBase; +import com.github.sonus21.rqueue.CoreUnitTest; +import com.github.sonus21.rqueue.core.RqueueMessage; +import com.github.sonus21.rqueue.core.RqueueMessageTemplate; +import com.github.sonus21.rqueue.core.spi.Capabilities; +import com.github.sonus21.rqueue.listener.QueueDetail; +import com.github.sonus21.rqueue.models.MessageMoveResult; +import com.github.sonus21.rqueue.utils.TestUtils; +import java.time.Duration; +import java.util.Collections; +import java.util.List; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.springframework.data.redis.connection.Message; +import org.springframework.data.redis.connection.MessageListener; +import org.springframework.data.redis.core.ListOperations; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.data.redis.listener.ChannelTopic; +import org.springframework.data.redis.listener.RedisMessageListenerContainer; +import org.springframework.data.redis.listener.Topic; + +/** + * Verifies that {@link RedisMessageBroker} delegates each SPI method to the same {@link + * RqueueMessageTemplate} call site that the existing public API uses, locking the delegation + * contract for Phase 2. + */ +@CoreUnitTest +class RedisMessageBrokerDelegationTest extends TestBase { + + private static final QueueDetail QUEUE = TestUtils.createQueueDetail("phase1-broker-test"); + + @Mock + private RqueueMessageTemplate template; + + @Mock + private RedisTemplate redisTemplate; + + @Mock + private ListOperations listOperations; + + @Mock + private RedisMessageListenerContainer pubSubContainer; + + private RedisMessageBroker broker; + + @BeforeEach + void setUp() { + MockitoAnnotations.openMocks(this); + broker = new RedisMessageBroker(template, pubSubContainer); + } + + @Test + void enqueueDelegatesToAddMessage() { + RqueueMessage m = RqueueMessage.builder().id("a").message("msg").build(); + when(template.addMessage(QUEUE.getQueueName(), m)).thenReturn(1L); + + broker.enqueue(QUEUE, m); + + verify(template).addMessage(QUEUE.getQueueName(), m); + } + + @Test + void enqueueWithDelayDelegatesToAddMessageWithDelay() { + RqueueMessage m = RqueueMessage.builder().id("a").message("msg").build(); + broker.enqueueWithDelay(QUEUE, m, 5000L); + + verify(template) + .addMessageWithDelay( + QUEUE.getScheduledQueueName(), QUEUE.getScheduledQueueChannelName(), m); + } + + @Test + void popDelegatesToTemplatePop() { + when(template.pop(any(), any(), any(), anyLong(), anyInt())) + .thenReturn(Collections.emptyList()); + + List out = broker.pop(QUEUE, "consumer", 5, Duration.ofSeconds(1)); + + assertNotNull(out); + verify(template) + .pop( + QUEUE.getQueueName(), + QUEUE.getProcessingQueueName(), + QUEUE.getProcessingQueueChannelName(), + QUEUE.getVisibilityTimeout(), + 5); + } + + @Test + void ackDelegatesToRemoveElementFromZset() { + RqueueMessage m = RqueueMessage.builder().id("a").message("msg").build(); + when(template.removeElementFromZset(QUEUE.getProcessingQueueName(), m)).thenReturn(1L); + + assertTrue(broker.ack(QUEUE, m)); + verify(template).removeElementFromZset(QUEUE.getProcessingQueueName(), m); + } + + @Test + void ackReturnsFalseWhenNotRemoved() { + RqueueMessage m = RqueueMessage.builder().id("a").message("msg").build(); + when(template.removeElementFromZset(QUEUE.getProcessingQueueName(), m)).thenReturn(0L); + + assertFalse(broker.ack(QUEUE, m)); + } + + @Test + void nackWithNoDelayDelegatesToMoveMessage() { + RqueueMessage m = RqueueMessage.builder().id("a").message("msg").build(); + assertTrue(broker.nack(QUEUE, m, 0L)); + verify(template).moveMessage(QUEUE.getProcessingQueueName(), QUEUE.getQueueName(), m, m); + } + + @Test + void nackWithDelayDelegatesToMoveMessageWithDelay() { + RqueueMessage m = RqueueMessage.builder().id("a").message("msg").build(); + assertTrue(broker.nack(QUEUE, m, 1500L)); + verify(template) + .moveMessageWithDelay( + QUEUE.getProcessingQueueName(), QUEUE.getScheduledQueueName(), m, m, 1500L); + } + + @Test + void moveExpiredDelegatesToMoveMessageZsetToList() { + when(template.moveMessageZsetToList( + eq(QUEUE.getScheduledQueueName()), eq(QUEUE.getQueueName()), eq(10))) + .thenReturn(new MessageMoveResult(7, true)); + + long moved = broker.moveExpired(QUEUE, System.currentTimeMillis(), 10); + + assertEquals(7L, moved); + verify(template).moveMessageZsetToList(QUEUE.getScheduledQueueName(), QUEUE.getQueueName(), 10); + } + + @Test + void peekDelegatesToReadFromList() { + when(template.readFromList(QUEUE.getQueueName(), 0L, 4L)).thenReturn(Collections.emptyList()); + + broker.peek(QUEUE, 0L, 5L); + + verify(template).readFromList(QUEUE.getQueueName(), 0L, 4L); + } + + @Test + void sizeUsesUnderlyingListOps() { + when(template.getTemplate()).thenReturn(redisTemplate); + when(redisTemplate.opsForList()).thenReturn(listOperations); + when(listOperations.size(QUEUE.getQueueName())).thenReturn(42L); + + assertEquals(42L, broker.size(QUEUE)); + verify(listOperations).size(QUEUE.getQueueName()); + } + + @Test + void publishUsesUnderlyingRedisTemplate() { + when(template.getTemplate()).thenReturn(redisTemplate); + + broker.publish("test-channel", "hello"); + + verify(redisTemplate).convertAndSend("test-channel", "hello"); + } + + @Test + void subscribeRegistersListenerOnContainerAndCloseRemovesIt() throws Exception { + final String[] received = new String[1]; + AutoCloseable handle = broker.subscribe("ch", s -> received[0] = s); + + // capture listener via verify + org.mockito.ArgumentCaptor listenerCaptor = + org.mockito.ArgumentCaptor.forClass(MessageListener.class); + org.mockito.ArgumentCaptor topicCaptor = + org.mockito.ArgumentCaptor.forClass(Topic.class); + verify(pubSubContainer).addMessageListener(listenerCaptor.capture(), topicCaptor.capture()); + assertEquals( + new ChannelTopic("ch").getTopic(), ((ChannelTopic) topicCaptor.getValue()).getTopic()); + + // simulate a delivered message + Message message = new Message() { + @Override + public byte[] getBody() { + return "payload".getBytes(); + } + + @Override + public byte[] getChannel() { + return "ch".getBytes(); + } + }; + listenerCaptor.getValue().onMessage(message, null); + assertEquals("payload", received[0]); + + handle.close(); + verify(pubSubContainer, times(1)) + .removeMessageListener(listenerCaptor.getValue(), topicCaptor.getValue()); + } + + @Test + void capabilitiesAreRedisDefaults() { + assertEquals(Capabilities.REDIS_DEFAULTS, broker.capabilities()); + assertTrue(broker.capabilities().supportsDelayedEnqueue()); + assertTrue(broker.capabilities().supportsScheduledIntrospection()); + assertTrue(broker.capabilities().supportsCronJobs()); + assertTrue(broker.capabilities().usesPrimaryHandlerDispatch()); + } + + @Test + void noBrokerStateLeaksToTemplateWhenUnused() { + // Constructing the broker alone must not call into the template. + new RedisMessageBroker(template, pubSubContainer); + verifyNoInteractions(template); + } +} diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/ConcurrentListenerTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/ConcurrentListenerTest.java index 539832ec3..5e0984eb3 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/ConcurrentListenerTest.java +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/ConcurrentListenerTest.java @@ -37,8 +37,8 @@ import com.github.sonus21.rqueue.dao.RqueueSystemConfigDao; import com.github.sonus21.rqueue.models.db.MessageMetadata; import com.github.sonus21.rqueue.models.enums.MessageStatus; +import com.github.sonus21.rqueue.service.RqueueMessageMetadataService; import com.github.sonus21.rqueue.utils.TimeoutUtils; -import com.github.sonus21.rqueue.web.service.RqueueMessageMetadataService; import java.util.HashMap; import java.util.LinkedList; import java.util.List; diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/JobImplTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/JobImplTest.java index 0bdf16d66..eccfefee7 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/JobImplTest.java +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/JobImplTest.java @@ -39,7 +39,7 @@ import com.github.sonus21.rqueue.config.RqueueConfig; import com.github.sonus21.rqueue.core.DefaultRqueueMessageConverter; import com.github.sonus21.rqueue.core.RqueueMessage; -import com.github.sonus21.rqueue.core.RqueueMessageTemplate; +import com.github.sonus21.rqueue.core.spi.MessageBroker; import com.github.sonus21.rqueue.core.support.RqueueMessageUtils; import com.github.sonus21.rqueue.dao.RqueueJobDao; import com.github.sonus21.rqueue.models.db.Execution; @@ -48,9 +48,9 @@ import com.github.sonus21.rqueue.models.enums.ExecutionStatus; import com.github.sonus21.rqueue.models.enums.JobStatus; import com.github.sonus21.rqueue.models.enums.MessageStatus; +import com.github.sonus21.rqueue.service.RqueueMessageMetadataService; import com.github.sonus21.rqueue.utils.RqueueMessageTestUtils; import com.github.sonus21.rqueue.utils.TestUtils; -import com.github.sonus21.rqueue.web.service.RqueueMessageMetadataService; import java.time.Duration; import org.apache.commons.lang3.reflect.FieldUtils; import org.junit.jupiter.api.BeforeEach; @@ -84,7 +84,7 @@ class JobImplTest extends TestBase { private RqueueJobDao rqueueJobDao; @Mock - private RqueueMessageTemplate rqueueMessageTemplate; + private MessageBroker messageBroker; @Mock private RqueueLockManager rqueueLockManager; @@ -109,7 +109,7 @@ private JobImpl instance() { rqueueConfig, messageMetadataService, rqueueJobDao, - rqueueMessageTemplate, + messageBroker, rqueueLockManager, queueDetail, messageMetadata, @@ -257,20 +257,16 @@ void updateExecutionTime() { void getVisibilityTimeout() { JobImpl job = instance(); job.execute(); - doReturn(-10L) - .when(rqueueMessageTemplate) - .getScore(queueDetail.getProcessingQueueName(), rqueueMessage); + doReturn(-10L).when(messageBroker).getVisibilityTimeoutScore(queueDetail, rqueueMessage); assertEquals(Duration.ZERO, job.getVisibilityTimeout()); doReturn(System.currentTimeMillis() + 10_000L) - .when(rqueueMessageTemplate) - .getScore(queueDetail.getProcessingQueueName(), rqueueMessage); + .when(messageBroker) + .getVisibilityTimeoutScore(queueDetail, rqueueMessage); Duration timeout = job.getVisibilityTimeout(); assertTrue(timeout.toMillis() <= 10_000 && timeout.toMillis() >= 9_000); - doReturn(0L) - .when(rqueueMessageTemplate) - .getScore(queueDetail.getProcessingQueueName(), rqueueMessage); + doReturn(0L).when(messageBroker).getVisibilityTimeoutScore(queueDetail, rqueueMessage); assertEquals(Duration.ZERO, job.getVisibilityTimeout()); } @@ -278,13 +274,9 @@ void getVisibilityTimeout() { void updateVisibilityTimeout() { JobImpl job = instance(); job.execute(); - doReturn(true) - .when(rqueueMessageTemplate) - .addScore(queueDetail.getProcessingQueueName(), rqueueMessage, 5_000L); + doReturn(true).when(messageBroker).extendVisibilityTimeout(queueDetail, rqueueMessage, 5_000L); assertTrue(job.updateVisibilityTimeout(Duration.ofSeconds(5))); - doReturn(false) - .when(rqueueMessageTemplate) - .addScore(queueDetail.getProcessingQueueName(), rqueueMessage, 5_000L); + doReturn(false).when(messageBroker).extendVisibilityTimeout(queueDetail, rqueueMessage, 5_000L); assertFalse(job.updateVisibilityTimeout(Duration.ofSeconds(5))); } diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/PriorityGroupListenerTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/PriorityGroupListenerTest.java index 543ff927f..f0658fa5b 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/PriorityGroupListenerTest.java +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/PriorityGroupListenerTest.java @@ -35,9 +35,9 @@ import com.github.sonus21.rqueue.listener.RqueueMessageListenerContainerTest.TestEventBroadcaster; import com.github.sonus21.rqueue.models.db.QueueConfig; import com.github.sonus21.rqueue.models.enums.PriorityMode; +import com.github.sonus21.rqueue.service.RqueueMessageMetadataService; import com.github.sonus21.rqueue.utils.TestUtils; import com.github.sonus21.rqueue.utils.TimeoutUtils; -import com.github.sonus21.rqueue.web.service.RqueueMessageMetadataService; import com.github.sonus21.test.TestTaskExecutor; import java.util.Arrays; import java.util.Collections; diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/RqueueExecutorTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/RqueueExecutorTest.java index 046cf4d05..953271599 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/RqueueExecutorTest.java +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/RqueueExecutorTest.java @@ -18,6 +18,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; @@ -35,7 +36,7 @@ import com.github.sonus21.rqueue.core.Job; import com.github.sonus21.rqueue.core.RqueueBeanProvider; import com.github.sonus21.rqueue.core.RqueueMessage; -import com.github.sonus21.rqueue.core.RqueueMessageTemplate; +import com.github.sonus21.rqueue.core.spi.MessageBroker; import com.github.sonus21.rqueue.core.support.MessageProcessor; import com.github.sonus21.rqueue.core.support.RqueueMessageUtils; import com.github.sonus21.rqueue.dao.RqueueJobDao; @@ -43,22 +44,19 @@ import com.github.sonus21.rqueue.listener.RqueueMessageListenerContainer.QueueStateMgr; import com.github.sonus21.rqueue.models.db.MessageMetadata; import com.github.sonus21.rqueue.models.enums.MessageStatus; +import com.github.sonus21.rqueue.service.RqueueMessageMetadataService; import com.github.sonus21.rqueue.utils.Constants; import com.github.sonus21.rqueue.utils.QueueThreadPool; import com.github.sonus21.rqueue.utils.RqueueMessageTestUtils; import com.github.sonus21.rqueue.utils.TestUtils; import com.github.sonus21.rqueue.utils.backoff.FixedTaskExecutionBackOff; import com.github.sonus21.rqueue.utils.backoff.TaskExecutionBackOff; -import com.github.sonus21.rqueue.web.service.RqueueMessageMetadataService; -import java.util.Collections; import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mock; import org.mockito.MockitoAnnotations; import org.springframework.context.ApplicationEventPublisher; -import org.springframework.data.redis.core.RedisCallback; -import org.springframework.data.redis.core.RedisTemplate; import org.springframework.messaging.MessagingException; import org.springframework.messaging.converter.MessageConverter; @@ -98,14 +96,11 @@ class RqueueExecutorTest extends TestBase { @Mock private RqueueSystemConfigDao rqueueSystemConfigDao; - @Mock - private RedisTemplate redisTemplate; - @Mock private ApplicationEventPublisher applicationEventPublisher; @Mock - private RqueueMessageTemplate messageTemplate; + private MessageBroker messageBroker; private RqueueMessage rqueueMessage = new RqueueMessage(); private PostProcessingHandler postProcessingHandler; @@ -130,10 +125,11 @@ public void init() throws IllegalAccessException { postProcessingHandler = new PostProcessingHandler( rqueueWebConfig, applicationEventPublisher, - messageTemplate, + messageBroker, taskBackOff, messageProcessorHandler, rqueueSystemConfigDao); + doReturn(messageBroker).when(rqueueBeanProvider).getMessageBroker(); doReturn(rqueueMessageMetadataService) .when(rqueueBeanProvider) .getRqueueMessageMetadataService(); @@ -175,7 +171,6 @@ void callDiscardProcessor() { @Test void callDeadLetterProcessor() { doReturn(true).when(rqueueLockManager).acquireLock(anyString(), anyString(), any()); - doReturn(redisTemplate).when(messageTemplate).getTemplate(); doReturn(preProcessMessageProcessor).when(rqueueBeanProvider).getPreExecutionMessageProcessor(); doThrow(new MessagingException("Failing for some reason.")) .when(messageHandler) @@ -187,9 +182,6 @@ void callDeadLetterProcessor() { doReturn(defaultMessageMetadata) .when(rqueueMessageMetadataService) .get(defaultMessageMetadata.getId()); - doReturn(Collections.emptyList()) - .when(redisTemplate) - .executePipelined(any(RedisCallback.class)); doReturn(3).when(rqueueConfig).getRetryPerPoll(); new RqueueExecutor( rqueueBeanProvider, @@ -224,13 +216,9 @@ void messageIsParkedForRetry() { queueDetail, queueThreadPool) .run(); - verify(messageTemplate, times(1)) - .moveMessageWithDelay( - eq(queueDetail.getProcessingQueueName()), - eq(queueDetail.getScheduledQueueName()), - eq(rqueueMessage), - any(), - eq(5000L)); + verify(messageBroker, times(1)) + .parkForRetry( + eq(queueDetail), any(RqueueMessage.class), any(RqueueMessage.class), eq(5000L)); } @Test @@ -307,8 +295,7 @@ void handleIgnoredMessage() { queueThreadPool) .run(); verify(messageHandler, times(0)).handleMessage(any()); - verify(messageTemplate, times(1)) - .removeElementFromZset(queueDetail.getProcessingQueueName(), rqueueMessage); + verify(messageBroker, times(1)).ack(eq(queueDetail), any(RqueueMessage.class)); } @Test @@ -333,7 +320,6 @@ void handlePeriodicMessage() { + newMessage.getProcessAt(); doReturn(1).when(rqueueConfig).getRetryPerPoll(); doReturn(preProcessMessageProcessor).when(rqueueBeanProvider).getPreExecutionMessageProcessor(); - doReturn(messageTemplate).when(rqueueBeanProvider).getRqueueMessageTemplate(); doReturn(defaultMessageMetadata) .when(rqueueMessageMetadataService) .getOrCreateMessageMetadata(any()); @@ -350,9 +336,8 @@ void handlePeriodicMessage() { queueDetail, queueThreadPool) .run(); - verify(messageTemplate, times(1)) - .scheduleMessage( - eq(queueDetail.getScheduledQueueName()), eq(messageKey), eq(newMessage), any()); + verify(messageBroker, times(1)) + .scheduleNext(eq(queueDetail), eq(messageKey), eq(newMessage), anyLong()); verify(messageHandler, times(1)).handleMessage(any()); } diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/RqueueMessageHandlerSkipPrimaryValidationTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/RqueueMessageHandlerSkipPrimaryValidationTest.java new file mode 100644 index 000000000..52fbc7bf2 --- /dev/null +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/RqueueMessageHandlerSkipPrimaryValidationTest.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ +package com.github.sonus21.rqueue.listener; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.github.sonus21.TestBase; +import com.github.sonus21.rqueue.CoreUnitTest; +import com.github.sonus21.rqueue.annotation.RqueueHandler; +import com.github.sonus21.rqueue.annotation.RqueueListener; +import com.github.sonus21.rqueue.core.DefaultRqueueMessageConverter; +import org.junit.jupiter.api.Test; +import org.springframework.context.support.StaticApplicationContext; + +@CoreUnitTest +class RqueueMessageHandlerSkipPrimaryValidationTest extends TestBase { + + @RqueueListener(value = "skip-validation-q") + static class HandlerWithoutPrimary { + @RqueueHandler + public void m1(String s) {} + + @RqueueHandler + public void m2(String s) {} + } + + /** Subclass that flips the capability flag before super.afterPropertiesSet(). */ + static class NatsLikeHandler extends RqueueMessageHandler { + NatsLikeHandler() { + super(new DefaultRqueueMessageConverter(), true); + setPrimaryHandlerDispatchEnabled(false); + } + } + + @Test + void skipsPrimaryValidationWhenCapabilityDisabled() { + StaticApplicationContext ctx = new StaticApplicationContext(); + ctx.registerSingleton("h", HandlerWithoutPrimary.class); + ctx.registerSingleton("rqueueMessageHandler", NatsLikeHandler.class); + // Without the flag this would throw because there's no primary among multiple @RqueueHandler. + assertDoesNotThrow(ctx::refresh); + NatsLikeHandler handler = ctx.getBean("rqueueMessageHandler", NatsLikeHandler.class); + assertFalse(handler.isPrimaryHandlerDispatchEnabled()); + } + + @Test + void defaultEnabledFlagIsTrue() { + RqueueMessageHandler h = new RqueueMessageHandler(new DefaultRqueueMessageConverter()); + assertTrue(h.isPrimaryHandlerDispatchEnabled()); + } +} diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/RqueueMessageListenerContainerBrokerBranchTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/RqueueMessageListenerContainerBrokerBranchTest.java new file mode 100644 index 000000000..6cc81e1e9 --- /dev/null +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/RqueueMessageListenerContainerBrokerBranchTest.java @@ -0,0 +1,275 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ +package com.github.sonus21.rqueue.listener; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.github.sonus21.TestBase; +import com.github.sonus21.rqueue.CoreUnitTest; +import com.github.sonus21.rqueue.annotation.RqueueListener; +import com.github.sonus21.rqueue.common.RqueueLockManager; +import com.github.sonus21.rqueue.config.RqueueConfig; +import com.github.sonus21.rqueue.config.RqueueWebConfig; +import com.github.sonus21.rqueue.core.DefaultRqueueMessageConverter; +import com.github.sonus21.rqueue.core.EndpointRegistry; +import com.github.sonus21.rqueue.core.RqueueBeanProvider; +import com.github.sonus21.rqueue.core.RqueueMessage; +import com.github.sonus21.rqueue.core.RqueueMessageTemplate; +import com.github.sonus21.rqueue.core.spi.Capabilities; +import com.github.sonus21.rqueue.core.spi.MessageBroker; +import com.github.sonus21.rqueue.dao.RqueueSystemConfigDao; +import com.github.sonus21.rqueue.service.RqueueMessageMetadataService; +import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.context.support.StaticApplicationContext; +import org.springframework.data.redis.connection.RedisConnectionFactory; + +@CoreUnitTest +class RqueueMessageListenerContainerBrokerBranchTest extends TestBase { + + @Mock + private RedisConnectionFactory redisConnectionFactory; + + @Mock + private ApplicationEventPublisher applicationEventPublisher; + + @Mock + private RqueueMessageTemplate rqueueMessageTemplate; + + @Mock + private RqueueSystemConfigDao rqueueSystemConfigDao; + + @Mock + private RqueueMessageMetadataService rqueueMessageMetadataService; + + @Mock + private RqueueWebConfig rqueueWebConfig; + + @Mock + private RqueueLockManager rqueueLockManager; + + private RqueueBeanProvider beanProvider; + private RqueueMessageHandler messageHandler; + + static class BrokerListener { + final AtomicInteger received = new AtomicInteger(); + + @RqueueListener(value = "broker-q1", consumerName = "consumer-A") + public void onMessage(String payload) { + received.incrementAndGet(); + } + } + + @BeforeEach + void setUp() { + MockitoAnnotations.openMocks(this); + RqueueConfig rqueueConfig = new RqueueConfig(redisConnectionFactory, null, true, 1); + beanProvider = new RqueueBeanProvider(); + beanProvider.setRqueueConfig(rqueueConfig); + beanProvider.setRqueueSystemConfigDao(rqueueSystemConfigDao); + beanProvider.setApplicationEventPublisher(applicationEventPublisher); + beanProvider.setRqueueMessageTemplate(rqueueMessageTemplate); + beanProvider.setRqueueMessageMetadataService(rqueueMessageMetadataService); + beanProvider.setRqueueWebConfig(rqueueWebConfig); + beanProvider.setRqueueLockManager(rqueueLockManager); + StaticApplicationContext applicationContext = new StaticApplicationContext(); + applicationContext.registerSingleton("brokerListener", BrokerListener.class); + messageHandler = new RqueueMessageHandler(new DefaultRqueueMessageConverter()); + messageHandler.setApplicationContext(applicationContext); + messageHandler.afterPropertiesSet(); + } + + /** Counting broker that records pop calls but never returns messages. */ + static class CountingBroker implements MessageBroker, AutoCloseable { + final AtomicInteger popCalls = new AtomicInteger(); + final AtomicBoolean closed = new AtomicBoolean(); + volatile Duration lastWait; + private final Capabilities caps; + + CountingBroker(Capabilities caps) { + this.caps = caps; + } + + @Override + public void enqueue(QueueDetail q, RqueueMessage m) {} + + @Override + public void enqueueWithDelay(QueueDetail q, RqueueMessage m, long delayMs) {} + + @Override + public List pop(QueueDetail q, String consumerName, int batch, Duration wait) { + popCalls.incrementAndGet(); + lastWait = wait; + try { + Thread.sleep(20); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + } + return Collections.emptyList(); + } + + @Override + public boolean ack(QueueDetail q, RqueueMessage m) { + return true; + } + + @Override + public boolean nack(QueueDetail q, RqueueMessage m, long retryDelayMs) { + return true; + } + + @Override + public long moveExpired(QueueDetail q, long now, int batch) { + return 0; + } + + @Override + public List peek(QueueDetail q, long offset, long count) { + return Collections.emptyList(); + } + + @Override + public long size(QueueDetail q) { + return 0; + } + + @Override + public AutoCloseable subscribe(String channel, Consumer handler) { + return () -> {}; + } + + @Override + public void publish(String channel, String payload) {} + + @Override + public Capabilities capabilities() { + return caps; + } + + @Override + public void close() { + closed.set(true); + } + } + + @Test + void brokerWithPrimaryHandlerDispatchUsesNormalStartQueuePath() throws Exception { + EndpointRegistry.delete(); + // Capabilities with usesPrimaryHandlerDispatch=true (e.g. NATS after the refactor). + CountingBroker broker = new CountingBroker(new Capabilities(true, false, false, true)); + TrackingContainer container = new TrackingContainer(messageHandler); + container.setMessageBroker(broker); + container.afterPropertiesSet(); + container.start(); + try { + // All brokers now go through the normal startQueue/startGroup path. + assertTrue( + container.startQueueCalled.get() || container.startGroupCalled.get(), + "startQueue or startGroup should be invoked for any broker using primary handler" + + " dispatch"); + } finally { + container.stop(); + container.destroy(); + } + assertTrue(broker.closed.get(), "AutoCloseable broker should be closed on destroy"); + } + + @Test + void redisDefaultsBrokerAlsoUsesNormalStartQueuePath() throws Exception { + EndpointRegistry.delete(); + CountingBroker broker = new CountingBroker(Capabilities.REDIS_DEFAULTS); + TrackingContainer container = new TrackingContainer(messageHandler); + container.setMessageBroker(broker); + container.afterPropertiesSet(); + container.start(); + try { + // broker-q1 has no priority group, so startQueue is expected. + assertTrue( + container.startQueueCalled.get() || container.startGroupCalled.get(), + "legacy Redis-side wiring should run for REDIS_DEFAULTS capabilities"); + assertFalse( + container.startBrokerPollersCalled.get(), + "startBrokerPollers no longer exists; flag must remain false"); + } finally { + container.stop(); + container.destroy(); + } + } + + @Test + void pollerForwardsPollingIntervalAsBrokerFetchWait() throws Exception { + EndpointRegistry.delete(); + CountingBroker broker = new CountingBroker(new Capabilities(true, false, false, true)); + RqueueMessageListenerContainer container = + new RqueueMessageListenerContainer(messageHandler, rqueueMessageTemplate); + container.rqueueBeanProvider = beanProvider; + container.setMessageBroker(broker); + long pollingInterval = 137L; + container.setPollingInterval(pollingInterval); + container.afterPropertiesSet(); + container.start(); + try { + // Wait for the poller to issue at least one pop call. + long deadline = System.currentTimeMillis() + 2000; + while (broker.popCalls.get() == 0 && System.currentTimeMillis() < deadline) { + Thread.sleep(20); + } + } finally { + container.stop(); + container.destroy(); + } + assertTrue(broker.popCalls.get() > 0, "poller should have issued at least one pop call"); + Duration wait = broker.lastWait; + assertNotNull(wait, "broker should have received a wait duration"); + assertFalse(wait.isZero(), "wait must not be Duration.ZERO; should match pollingInterval"); + assertTrue( + wait.toMillis() == pollingInterval, + "wait should equal the configured pollingInterval (got " + wait + ")"); + } + + private class TrackingContainer extends RqueueMessageListenerContainer { + final AtomicBoolean startBrokerPollersCalled = new AtomicBoolean(); + final AtomicBoolean startQueueCalled = new AtomicBoolean(); + final AtomicBoolean startGroupCalled = new AtomicBoolean(); + + TrackingContainer(RqueueMessageHandler handler) { + super(handler, rqueueMessageTemplate); + this.rqueueBeanProvider = beanProvider; + } + + @Override + protected void startQueue(String pollerKey, QueueDetail queueDetail) { + startQueueCalled.set(true); + // Do not actually start the poller; it would need a real broker. + } + + @Override + protected void startGroup(String groupName, List queueDetails) { + startGroupCalled.set(true); + } + } +} diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/RqueueMessageListenerContainerPriorityTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/RqueueMessageListenerContainerPriorityTest.java new file mode 100644 index 000000000..aa7ee971a --- /dev/null +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/RqueueMessageListenerContainerPriorityTest.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ +package com.github.sonus21.rqueue.listener; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.github.sonus21.rqueue.CoreUnitTest; +import com.github.sonus21.rqueue.core.RqueueMessage; +import com.github.sonus21.rqueue.core.spi.Capabilities; +import com.github.sonus21.rqueue.core.spi.MessageBroker; +import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import org.junit.jupiter.api.Test; + +@CoreUnitTest +class RqueueMessageListenerContainerPriorityTest { + + @Test + void messageBrokerDefaultEnqueueDelegatesToUnsuffixedOverload() { + final AtomicInteger plain = new AtomicInteger(); + MessageBroker broker = new MessageBroker() { + @Override + public void enqueue(QueueDetail q, RqueueMessage m) { + plain.incrementAndGet(); + } + + @Override + public void enqueueWithDelay(QueueDetail q, RqueueMessage m, long delayMs) {} + + @Override + public List pop(QueueDetail q, String consumerName, int batch, Duration wait) { + return Collections.emptyList(); + } + + @Override + public boolean ack(QueueDetail q, RqueueMessage m) { + return true; + } + + @Override + public boolean nack(QueueDetail q, RqueueMessage m, long retryDelayMs) { + return true; + } + + @Override + public long moveExpired(QueueDetail q, long now, int batch) { + return 0; + } + + @Override + public List peek(QueueDetail q, long offset, long count) { + return Collections.emptyList(); + } + + @Override + public long size(QueueDetail q) { + return 0; + } + + @Override + public AutoCloseable subscribe(String channel, Consumer handler) { + return () -> {}; + } + + @Override + public void publish(String channel, String payload) {} + + @Override + public Capabilities capabilities() { + return Capabilities.REDIS_DEFAULTS; + } + }; + QueueDetail qd = QueueDetail.builder() + .name("q1") + .queueName("__rq::queue::q1") + .processingQueueName("__rq::pq::q1") + .completedQueueName("__rq::cq::q1") + .scheduledQueueName("__rq::sq::q1") + .processingQueueChannelName("__rq::ch::q1") + .scheduledQueueChannelName("__rq::sch::q1") + .visibilityTimeout(30000) + .numRetry(3) + .priority(Collections.emptyMap()) + .build(); + RqueueMessage msg = + RqueueMessage.builder().id("x").queueName("q1").message("p").build(); + broker.enqueue(qd, "high", msg); + broker.enqueue(qd, msg); + assertEquals(2, plain.get(), "default priority overload should delegate to enqueue(q, m)"); + } +} diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/RqueueMessageListenerContainerTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/RqueueMessageListenerContainerTest.java index 34346d660..6053ee9dd 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/RqueueMessageListenerContainerTest.java +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/RqueueMessageListenerContainerTest.java @@ -41,10 +41,10 @@ import com.github.sonus21.rqueue.models.db.QueueConfig; import com.github.sonus21.rqueue.models.enums.MessageStatus; import com.github.sonus21.rqueue.models.event.RqueueBootstrapEvent; +import com.github.sonus21.rqueue.service.RqueueMessageMetadataService; import com.github.sonus21.rqueue.utils.Constants; import com.github.sonus21.rqueue.utils.TestUtils; import com.github.sonus21.rqueue.utils.TimeoutUtils; -import com.github.sonus21.rqueue.web.service.RqueueMessageMetadataService; import com.github.sonus21.test.TestTaskExecutor; import io.lettuce.core.RedisCommandExecutionException; import java.util.Collections; diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/RqueueMiddlewareTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/RqueueMiddlewareTest.java index ea0103d1a..7b9b6ced4 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/RqueueMiddlewareTest.java +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/RqueueMiddlewareTest.java @@ -22,6 +22,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.times; @@ -36,7 +37,6 @@ import com.github.sonus21.rqueue.core.Job; import com.github.sonus21.rqueue.core.RqueueBeanProvider; import com.github.sonus21.rqueue.core.RqueueMessage; -import com.github.sonus21.rqueue.core.RqueueMessageTemplate; import com.github.sonus21.rqueue.core.context.Context; import com.github.sonus21.rqueue.core.context.DefaultContext; import com.github.sonus21.rqueue.core.middleware.ContextMiddleware; @@ -44,6 +44,7 @@ import com.github.sonus21.rqueue.core.middleware.PermissionMiddleware; import com.github.sonus21.rqueue.core.middleware.ProfilerMiddleware; import com.github.sonus21.rqueue.core.middleware.RateLimiterMiddleware; +import com.github.sonus21.rqueue.core.spi.MessageBroker; import com.github.sonus21.rqueue.core.support.MessageProcessor; import com.github.sonus21.rqueue.core.support.RqueueMessageUtils; import com.github.sonus21.rqueue.dao.RqueueJobDao; @@ -52,6 +53,7 @@ import com.github.sonus21.rqueue.listener.RqueueMessageListenerContainer.QueueStateMgr; import com.github.sonus21.rqueue.models.db.MessageMetadata; import com.github.sonus21.rqueue.models.enums.MessageStatus; +import com.github.sonus21.rqueue.service.RqueueMessageMetadataService; import com.github.sonus21.rqueue.utils.Constants; import com.github.sonus21.rqueue.utils.QueueThreadPool; import com.github.sonus21.rqueue.utils.RqueueMessageTestUtils; @@ -59,7 +61,6 @@ import com.github.sonus21.rqueue.utils.TimeoutUtils; import com.github.sonus21.rqueue.utils.backoff.FixedTaskExecutionBackOff; import com.github.sonus21.rqueue.utils.backoff.TaskExecutionBackOff; -import com.github.sonus21.rqueue.web.service.RqueueMessageMetadataService; import com.google.common.util.concurrent.RateLimiter; import java.time.Duration; import java.util.ArrayList; @@ -108,7 +109,7 @@ class RqueueMiddlewareTest extends TestBase { private RqueueJobDao rqueueJobDao; @Mock - private RqueueMessageTemplate messageTemplate; + private MessageBroker messageBroker; @Mock private RqueueMessageHandler messageHandler; @@ -141,10 +142,11 @@ public void init() throws IllegalAccessException { postProcessingHandler = new PostProcessingHandler( rqueueWebConfig, applicationEventPublisher, - messageTemplate, + messageBroker, taskBackOff, messageProcessorHandler, rqueueSystemConfigDao); + doReturn(messageBroker).when(rqueueBeanProvider).getMessageBroker(); doReturn(rqueueMessageMetadataService) .when(rqueueBeanProvider) .getRqueueMessageMetadataService(); @@ -185,8 +187,7 @@ void logMiddleware() { queueThreadPool) .run(); verify(messageHandler, times(1)).handleMessage(any()); - verify(messageTemplate, times(1)) - .removeElementFromZset(queueDetail.getProcessingQueueName(), rqueueMessage); + verify(messageBroker, times(1)).ack(eq(queueDetail), any(RqueueMessage.class)); assertEquals(1, logMiddleware.jobs.size()); } @@ -213,8 +214,7 @@ void logAndContextMiddleware() { queueThreadPool) .run(); verify(messageHandler, times(1)).handleMessage(any()); - verify(messageTemplate, times(1)) - .removeElementFromZset(queueDetail.getProcessingQueueName(), rqueueMessage); + verify(messageBroker, times(1)).ack(eq(queueDetail), any(RqueueMessage.class)); assertEquals(1, logMiddleware.jobs.size()); assertEquals(1, contextMiddleware.jobs.size()); assertNotNull(logMiddleware.jobs.get(0).getId()); @@ -275,8 +275,8 @@ void logContextAndPermissionMiddleware() { .run(); verify(messageHandler, times(1)).handleMessage(any()); - verify(messageTemplate, times(1)) - .removeElementFromZset(queueDetail.getProcessingQueueName(), rqueueMessage); + // Both executors call ack: allowed message after success, declined message after IGNORED. + verify(messageBroker, times(2)).ack(eq(queueDetail), any(RqueueMessage.class)); assertEquals(2, logMiddleware.jobs.size()); assertEquals(2, contextMiddleware.jobs.size()); assertEquals(2, permissionMiddleware.jobs.size()); @@ -377,8 +377,7 @@ void logAndProfilerMiddleware() { queueThreadPool) .run(); verify(messageHandler, times(1)).handleMessage(any()); - verify(messageTemplate, times(1)) - .removeElementFromZset(queueDetail.getProcessingQueueName(), rqueueMessage); + verify(messageBroker, times(1)).ack(eq(queueDetail), any(RqueueMessage.class)); assertEquals(1, logMiddleware.jobs.size()); assertEquals(1, profilerMiddleware.jobs.size()); assertEquals(rqueueMessage.getId(), profilerMiddleware.jobs.get(0).getMessageId()); diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/metrics/RqueueMetricsTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/metrics/RqueueMetricsTest.java index 08b24e899..683f8dc1e 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/metrics/RqueueMetricsTest.java +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/metrics/RqueueMetricsTest.java @@ -18,16 +18,14 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import com.github.sonus21.TestBase; import com.github.sonus21.rqueue.CoreUnitTest; import com.github.sonus21.rqueue.config.MetricsProperties; import com.github.sonus21.rqueue.core.EndpointRegistry; -import com.github.sonus21.rqueue.dao.RqueueStringDao; import com.github.sonus21.rqueue.listener.QueueDetail; import com.github.sonus21.rqueue.models.event.RqueueBootstrapEvent; import com.github.sonus21.rqueue.utils.TestUtils; @@ -54,7 +52,7 @@ class RqueueMetricsTest extends TestBase { TestUtils.createQueueDetail(simpleQueue, deadLetterQueue); @Mock - private RqueueStringDao rqueueStringDao; + private RqueueQueueMetricsProvider queueMetricsProvider; @Mock private QueueCounter queueCounter; @@ -120,45 +118,24 @@ private RqueueMetrics rqueueMetrics( RqueueMetrics metrics = new RqueueMetrics(queueCounter); FieldUtils.writeField(metrics, "meterRegistry", meterRegistry, true); FieldUtils.writeField(metrics, "metricsProperties", metricsProperties, true); + FieldUtils.writeField(metrics, "queueMetricsProvider", queueMetricsProvider, true); return metrics; } @Test void queueStatistics() throws IllegalAccessException { - doAnswer(invocation -> { - String zsetName = invocation.getArgument(0); - if (zsetName.equals(scheduledQueueDetail.getScheduledQueueName())) { - return 5L; - } - if (zsetName.equals(simpleQueueDetail.getProcessingQueueName())) { - return 10L; - } - if (zsetName.equals(scheduledQueueDetail.getProcessingQueueName())) { - return 15L; - } - return null; - }) - .when(rqueueStringDao) - .getSortedSetSize(anyString()); + // All four gauges read through RqueueQueueMetricsProvider, keyed by user-facing queue name. + when(queueMetricsProvider.getPendingMessageCount(simpleQueue)).thenReturn(100L); + when(queueMetricsProvider.getPendingMessageCount(scheduledQueue)).thenReturn(200L); + when(queueMetricsProvider.getProcessingMessageCount(simpleQueue)).thenReturn(10L); + when(queueMetricsProvider.getProcessingMessageCount(scheduledQueue)).thenReturn(15L); + when(queueMetricsProvider.getScheduledMessageCount(simpleQueue)).thenReturn(0L); + when(queueMetricsProvider.getScheduledMessageCount(scheduledQueue)).thenReturn(5L); + // Only simpleQueue has a DLQ configured; the scheduledQueue gauge is never registered. + when(queueMetricsProvider.getDeadLetterMessageCount(simpleQueue)).thenReturn(300L); - doAnswer(invocation -> { - String listName = invocation.getArgument(0); - if (listName.equals(simpleQueueDetail.getQueueName())) { - return 100L; - } - if (listName.equals(scheduledQueueDetail.getQueueName())) { - return 200L; - } - if (listName.equals(deadLetterQueue)) { - return 300L; - } - return null; - }) - .when(rqueueStringDao) - .getListSize(anyString()); MeterRegistry meterRegistry = new SimpleMeterRegistry(); RqueueMetrics metrics = rqueueMetrics(meterRegistry, metricsProperties); - FieldUtils.writeField(metrics, "rqueueStringDao", rqueueStringDao, true); metrics.onApplicationEvent(new RqueueBootstrapEvent("Test", true)); verifyQueueStatistics(meterRegistry, simpleQueue, 100, 10, 300, 0); verifyQueueStatistics(meterRegistry, scheduledQueue, 200, 15, 0, 5); diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/metrics/RqueueQueueMetricsTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/metrics/RqueueQueueMetricsTest.java deleted file mode 100644 index bd07a6cb5..000000000 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/metrics/RqueueQueueMetricsTest.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (c) 2021-2026 Sonu Kumar - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * You may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and limitations under the License. - * - */ - -package com.github.sonus21.rqueue.metrics; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; - -import com.github.sonus21.TestBase; -import com.github.sonus21.rqueue.CoreUnitTest; -import com.github.sonus21.rqueue.common.RqueueRedisTemplate; -import com.github.sonus21.rqueue.core.EndpointRegistry; -import com.github.sonus21.rqueue.listener.QueueDetail; -import com.github.sonus21.rqueue.utils.PriorityUtils; -import com.github.sonus21.rqueue.utils.TestUtils; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; - -@CoreUnitTest -class RqueueQueueMetricsTest extends TestBase { - - private static final String queueName = "test"; - private static final String priorityName = "high"; - private static final QueueDetail queueDetail = TestUtils.createQueueDetail(queueName); - private static final QueueDetail queueDetail2 = - TestUtils.createQueueDetail(PriorityUtils.getQueueNameForPriority(queueName, priorityName)); - private final RqueueRedisTemplate redisTemplate = mock(RqueueRedisTemplate.class); - private final RqueueQueueMetrics queueMetrics = new RqueueQueueMetrics(redisTemplate); - - @BeforeAll - static void setUp() { - EndpointRegistry.delete(); - EndpointRegistry.register(queueDetail); - EndpointRegistry.register(queueDetail2); - } - - @AfterAll - static void tearDown() { - EndpointRegistry.delete(); - } - - @Test - void getPendingMessageCount() { - doReturn(100L).when(redisTemplate).getListSize(queueDetail.getQueueName()); - assertEquals(100L, queueMetrics.getPendingMessageCount(queueName)); - assertEquals(-1L, queueMetrics.getPendingMessageCount("unknown")); - } - - @Test - void getPendingMessageCountWithPriority() { - doReturn(100L).when(redisTemplate).getListSize(queueDetail2.getQueueName()); - assertEquals(100L, queueMetrics.getPendingMessageCount(queueName, priorityName)); - assertEquals(-1L, queueMetrics.getPendingMessageCount("unknown", priorityName)); - } - - @Test - void getScheduledMessageCount() { - doReturn(100L).when(redisTemplate).getZsetSize(queueDetail.getScheduledQueueName()); - assertEquals(100L, queueMetrics.getScheduledMessageCount(queueName)); - assertEquals(-1L, queueMetrics.getScheduledMessageCount("unknown")); - } - - @Test - void getScheduledMessageCountWithPriority() { - doReturn(100L).when(redisTemplate).getZsetSize(queueDetail2.getScheduledQueueName()); - assertEquals(100L, queueMetrics.getScheduledMessageCount(queueName, priorityName)); - assertEquals(-1L, queueMetrics.getScheduledMessageCount("unknown", priorityName)); - } - - @Test - void getProcessingMessageCount() { - doReturn(100L).when(redisTemplate).getZsetSize(queueDetail.getProcessingQueueName()); - assertEquals(100L, queueMetrics.getProcessingMessageCount(queueName)); - assertEquals(-1L, queueMetrics.getProcessingMessageCount("unknown")); - } - - @Test - void getProcessingMessageCountWithPriority() { - doReturn(100L).when(redisTemplate).getZsetSize(queueDetail2.getProcessingQueueName()); - assertEquals(100L, queueMetrics.getProcessingMessageCount(queueName, priorityName)); - assertEquals(-1L, queueMetrics.getProcessingMessageCount("unknown", priorityName)); - } -} diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/models/db/QueueStatisticsTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/models/db/QueueStatisticsTest.java index 343a006d1..24dc56ab2 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/models/db/QueueStatisticsTest.java +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/models/db/QueueStatisticsTest.java @@ -16,10 +16,12 @@ package com.github.sonus21.rqueue.models.db; +import static com.github.sonus21.rqueue.models.db.QueueStatisticsFixtures.addData; +import static com.github.sonus21.rqueue.models.db.QueueStatisticsFixtures.checkNonNull; +import static com.github.sonus21.rqueue.models.db.QueueStatisticsFixtures.validate; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; import com.github.sonus21.TestBase; import com.github.sonus21.rqueue.CoreUnitTest; @@ -31,34 +33,6 @@ public class QueueStatisticsTest extends TestBase { private final String id = "__rq::q-stat::slow-queue"; - public static void validate(QueueStatistics queueStatistics, int count) { - assertEquals(count, queueStatistics.getJobRunTime().size()); - assertEquals(count, queueStatistics.getTasksSuccessful().size()); - assertEquals(count, queueStatistics.getTasksDiscarded().size()); - assertEquals(count, queueStatistics.getTasksMovedToDeadLetter().size()); - assertEquals(count, queueStatistics.getTasksRetried().size()); - } - - public static void checkNonNull(QueueStatistics queueStatistics, String date) { - assertNotNull(queueStatistics.jobRunTime(date)); - assertTrue(queueStatistics.tasksSuccessful(date) > 0); - assertTrue(queueStatistics.tasksDiscarded(date) > 0); - assertTrue(queueStatistics.tasksMovedToDeadLetter(date) > 0); - assertTrue(queueStatistics.tasksRetried(date) > 0); - } - - public static void addData(QueueStatistics queueStatistics, LocalDate localDate, int day) { - String date = localDate.minusDays(day).toString(); - int val = 1 + (int) (Math.random() * 100); - queueStatistics.incrementSuccessful(date, val); - queueStatistics.incrementDiscard(date, val); - queueStatistics.incrementDeadLetter(date, val); - queueStatistics.incrementRetry(date, val); - int val2 = 1 + (int) (Math.random() * 100); - JobRunTime jobRunTime = new JobRunTime(val, val2, val * val2, val2); - queueStatistics.updateJobExecutionTime(date, jobRunTime); - } - @Test void incrementDeadLetter() { QueueStatistics queueStatistics = new QueueStatistics(id); diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/worker/RqueueWorkerRegistryImplTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/worker/RqueueWorkerRegistryImplTest.java deleted file mode 100644 index 87463e26f..000000000 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/worker/RqueueWorkerRegistryImplTest.java +++ /dev/null @@ -1,212 +0,0 @@ -/* - * Copyright (c) 2026 Sonu Kumar - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * You may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and limitations under the License. - * - */ - -package com.github.sonus21.rqueue.worker; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; - -import com.github.sonus21.TestBase; -import com.github.sonus21.rqueue.CoreUnitTest; -import com.github.sonus21.rqueue.common.RqueueRedisTemplate; -import com.github.sonus21.rqueue.config.RqueueConfig; -import com.github.sonus21.rqueue.listener.QueueDetail; -import com.github.sonus21.rqueue.models.Concurrency; -import com.github.sonus21.rqueue.models.registry.RqueueWorkerInfo; -import com.github.sonus21.rqueue.models.registry.RqueueWorkerPollerMetadata; -import com.github.sonus21.rqueue.models.registry.RqueueWorkerPollerView; -import com.github.sonus21.rqueue.utils.RedisUtils; -import com.github.sonus21.rqueue.utils.SerializationUtils; -import java.lang.reflect.Field; -import java.time.Duration; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; -import org.springframework.core.task.SimpleAsyncTaskExecutor; -import org.springframework.data.redis.connection.RedisConnectionFactory; -import org.springframework.data.redis.core.RedisTemplate; -import tools.jackson.databind.ObjectMapper; - -@CoreUnitTest -class RqueueWorkerRegistryImplTest extends TestBase { - - private final RedisConnectionFactory redisConnectionFactory = null; - private TestWorkerTemplate workerTemplate; - private TestStringTemplate stringTemplate; - private final RedisTemplate redisTemplate = new RedisTemplate<>(); - - private RedisUtils.RedisTemplateProvider originalRedisTemplateProvider; - private final ObjectMapper objectMapper = SerializationUtils.createObjectMapper(); - - @BeforeEach - void init() { - MockitoAnnotations.openMocks(this); - redisTemplate.setConnectionFactory(Mockito.mock(RedisConnectionFactory.class)); - originalRedisTemplateProvider = RedisUtils.redisTemplateProvider; - RedisUtils.redisTemplateProvider = new RedisUtils.RedisTemplateProvider() { - @Override - public RedisTemplate getRedisTemplate( - RedisConnectionFactory redisConnectionFactory) { - return (RedisTemplate) redisTemplate; - } - }; - workerTemplate = new TestWorkerTemplate(); - stringTemplate = new TestStringTemplate(); - } - - @AfterEach - void cleanup() { - RedisUtils.redisTemplateProvider = originalRedisTemplateProvider; - } - - @Test - void getQueueWorkersUsesCapacityExhaustedAsActivity() throws Exception { - RqueueConfig rqueueConfig = new RqueueConfig(redisConnectionFactory, null, false, 2); - rqueueConfig.setWorkerRegistryEnabled(true); - rqueueConfig.setWorkerRegistryQueueHeartbeatIntervalInSeconds(15); - rqueueConfig.setWorkerRegistryQueueTtlInSeconds(3600); - RqueueWorkerRegistryImpl registry = new RqueueWorkerRegistryImpl(rqueueConfig); - setField(registry, "workerTemplate", workerTemplate); - setField(registry, "stringTemplate", stringTemplate); - - long now = System.currentTimeMillis(); - String workerId = "worker-1"; - RqueueWorkerPollerMetadata metadata = RqueueWorkerPollerMetadata.builder() - .workerId(workerId) - .lastPollAt(now - Duration.ofSeconds(40).toMillis()) - .lastCapacityExhaustedAt(now) - .capacityExhaustedCount(2L) - .build(); - stringTemplate.values = - Collections.singletonMap(workerId, objectMapper.writeValueAsString(metadata)); - workerTemplate.values = Collections.singletonList(RqueueWorkerInfo.builder() - .workerId(workerId) - .host("host-1") - .pid("123") - .startedAt(now - Duration.ofMinutes(5).toMillis()) - .lastSeenAt(now) - .build()); - - List workers = registry.getQueueWorkers("test"); - - assertEquals(1, workers.size()); - assertEquals("ACTIVE", workers.get(0).getStatus()); - assertEquals(2L, workers.get(0).getCapacityExhaustedCount()); - assertEquals("host-1", workers.get(0).getHost()); - assertFalse(workers.get(0).getLastCapacityExhaustedAge().isEmpty()); - } - - @Test - void recordQueueCapacityExhaustedSaturatesAtLongMaxValue() throws Exception { - RqueueConfig rqueueConfig = new RqueueConfig(redisConnectionFactory, null, false, 2); - rqueueConfig.setWorkerRegistryEnabled(true); - rqueueConfig.setWorkerRegistryQueueHeartbeatIntervalInSeconds(15); - rqueueConfig.setWorkerRegistryQueueTtlInSeconds(3600); - RqueueWorkerRegistryImpl registry = new RqueueWorkerRegistryImpl(rqueueConfig); - setField(registry, "workerTemplate", workerTemplate); - setField(registry, "stringTemplate", stringTemplate); - setField( - registry, - "capacityExhaustedCountByQueue", - new java.util.concurrent.ConcurrentHashMap<>( - Collections.singletonMap("test-queue", Long.MAX_VALUE))); - - QueueDetail queueDetail = QueueDetail.builder() - .name("test-queue") - .queueName("test-queue") - .processingQueueName("test-queue-processing") - .processingQueueChannelName("test-queue-processing-channel") - .scheduledQueueName("test-queue-scheduled") - .scheduledQueueChannelName("test-queue-scheduled-channel") - .completedQueueName("test-queue-completed") - .active(true) - .visibilityTimeout(1000L) - .batchSize(1) - .numRetry(3) - .concurrency(new Concurrency(1, 1)) - .priority(Collections.emptyMap()) - .build(); - - registry.recordQueueCapacityExhausted( - queueDetail, - new com.github.sonus21.rqueue.utils.QueueThreadPool( - new SimpleAsyncTaskExecutor(), false, 1)); - - RqueueWorkerPollerMetadata metadata = - objectMapper.readValue(stringTemplate.lastHashValue, RqueueWorkerPollerMetadata.class); - - assertEquals(Long.MAX_VALUE, metadata.getCapacityExhaustedCount()); - } - - private static void setField(Object target, String fieldName, Object value) throws Exception { - Field field = target.getClass().getDeclaredField(fieldName); - field.setAccessible(true); - field.set(target, value); - } - - private static class TestWorkerTemplate extends RqueueRedisTemplate { - private List values = Collections.emptyList(); - private RqueueWorkerInfo lastValue; - - TestWorkerTemplate() { - super(null); - } - - @Override - public List mget(java.util.Collection keys) { - return values; - } - - @Override - public void set(String key, RqueueWorkerInfo val, Duration duration) { - lastValue = val; - } - } - - private static class TestStringTemplate extends RqueueRedisTemplate { - private Map values = Collections.emptyMap(); - private String lastHashValue; - - TestStringTemplate() { - super(null); - } - - @Override - public Map getHashEntries(String key) { - return values; - } - - @Override - public void putHashValue(String key, String hashKey, String val) { - lastHashValue = val; - } - - @Override - public Boolean expire(String key, Duration duration) { - return true; - } - - @Override - public Long deleteHashValues(String key, String... hashKeys) { - return 0L; - } - } -} diff --git a/rqueue-nats/build.gradle b/rqueue-nats/build.gradle new file mode 100644 index 000000000..fb2665cea --- /dev/null +++ b/rqueue-nats/build.gradle @@ -0,0 +1,56 @@ +plugins { + id 'com.vanniktech.maven.publish' version '0.28.0' +} +apply from: "${rootDir}/gradle/packaging.gradle" +apply from: "${rootDir}/gradle/test-runner.gradle" +apply from: "${rootDir}/gradle/code-publish.gradle" + +import com.vanniktech.maven.publish.SonatypeHost; + +mavenPublishing { + publishToMavenCentral(SonatypeHost.CENTRAL_PORTAL) + signAllPublications() + pom { + name = "Rqueue NATS" + description = "NATS / JetStream backend for Rqueue (preview). Add this jar and set rqueue.backend=nats to switch from Redis to NATS." + url = "https://github.com/sonus21/rqueue" + licenses { + license { + name = "Apache License 2.0" + url = "https://www.apache.org/licenses/LICENSE-2.0.txt" + } + } + developers { + developer { + id = "sonus21" + name = "Sonu Kumar" + email = "sonunitw12@gmail.com" + } + } + scm { + url = "https://github.com/sonus21/rqueue" + connection = "scm:git:git://github.com/sonus21/rqueue.git" + developerConnection = "scm:git:ssh://git@github.com:sonus21/rqueue.git" + } + issueManagement { + system = "GitHub" + url = "https://github.com/sonus21/rqueue/issues" + } + } +} + +dependencies { + // Broker-impl module only. No Spring / Spring Boot dependencies live here. + // Spring wiring is provided by rqueue-spring (@Configuration + @EnableRqueue) and + // rqueue-spring-boot-starter (auto-config), both of which reference this module's + // types via compileOnly and gate them behind @ConditionalOnClass(JetStream.class). + api project(":rqueue-core") + // NATS-shaped web service impls (NatsRqueueQDetailService, etc.) implement interfaces + // declared in rqueue-web. Mirrors how rqueue-redis pulls rqueue-web for the same reason. + api project(":rqueue-web") + api "io.nats:jnats:${natsVersion}" + testImplementation project(":rqueue-test-util") + testImplementation "org.testcontainers:testcontainers:${testcontainersVersion}" + testImplementation "org.testcontainers:junit-jupiter:${testcontainersVersion}" + testImplementation "io.projectreactor:reactor-test:${projectReactorReactorTestVersion}" +} diff --git a/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/RqueueNatsConfig.java b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/RqueueNatsConfig.java new file mode 100644 index 000000000..04bdf123f --- /dev/null +++ b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/RqueueNatsConfig.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ +package com.github.sonus21.rqueue.nats; + +import io.nats.client.api.RetentionPolicy; +import io.nats.client.api.StorageType; +import java.time.Duration; +import lombok.Getter; +import lombok.Setter; +import lombok.experimental.Accessors; + +/** + * Configuration POJO for the JetStream broker. Use {@link #defaults()} for sensible defaults and + * the fluent setters to override individual fields. Mutable on purpose - this class is constructed + * once at startup and read by the broker at runtime; thread-safety is the caller's responsibility. + */ +@Getter +@Setter +@Accessors(chain = true) +public class RqueueNatsConfig { + + private String streamPrefix = "rqueue-js-"; + private String subjectPrefix = "rqueue.js."; + private String dlqStreamSuffix = "-dlq"; + private String dlqSubjectSuffix = ".dlq"; + + private boolean autoCreateStreams = true; + private boolean autoCreateConsumers = true; + private boolean autoCreateDlqStream = false; + + private StreamDefaults streamDefaults = new StreamDefaults(); + private ConsumerDefaults consumerDefaults = new ConsumerDefaults(); + + /** Default fetch wait when {@code pop()} is called and no explicit wait is given. */ + private Duration defaultFetchWait = Duration.ofSeconds(2); + + public static RqueueNatsConfig defaults() { + return new RqueueNatsConfig(); + } + + // ---- nested defaults ---------------------------------------------------- + + @Getter + @Setter + @Accessors(chain = true) + public static class StreamDefaults { + private int replicas = 1; + private StorageType storage = StorageType.File; + private RetentionPolicy retention = RetentionPolicy.Limits; + private Duration duplicateWindow = Duration.ofMinutes(2); + private long maxMsgs = -1; + private long maxBytes = -1; + private Duration maxAge = null; + } + + @Getter + @Setter + @Accessors(chain = true) + public static class ConsumerDefaults { + private Duration ackWait = Duration.ofSeconds(30); + private long maxDeliver = 3; + private long maxAckPending = 1000; + } +} diff --git a/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/RqueueNatsException.java b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/RqueueNatsException.java new file mode 100644 index 000000000..6381e0b70 --- /dev/null +++ b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/RqueueNatsException.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ +package com.github.sonus21.rqueue.nats; + +/** + * Runtime wrapper for IOException, JetStreamApiException and other NATS errors thrown from the + * JetStream broker. The message always includes the queue or subject context that the call was + * targeting so operators can grep server-side logs. + */ +public class RqueueNatsException extends RuntimeException { + private static final long serialVersionUID = 1L; + + public RqueueNatsException(String message) { + super(message); + } + + public RqueueNatsException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/dao/NatsRqueueJobDao.java b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/dao/NatsRqueueJobDao.java new file mode 100644 index 000000000..c5c62eb34 --- /dev/null +++ b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/dao/NatsRqueueJobDao.java @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ + +package com.github.sonus21.rqueue.nats.dao; + +import com.github.sonus21.rqueue.config.NatsBackendCondition; +import com.github.sonus21.rqueue.dao.RqueueJobDao; +import com.github.sonus21.rqueue.models.db.RqueueJob; +import com.github.sonus21.rqueue.nats.internal.NatsProvisioner; +import com.github.sonus21.rqueue.nats.kv.NatsKvBuckets; +import io.nats.client.JetStreamApiException; +import io.nats.client.KeyValue; +import io.nats.client.api.KeyValueEntry; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.DependsOn; +import org.springframework.stereotype.Repository; + +/** + * NATS-backed {@link RqueueJobDao} using a JetStream KV bucket as the job store. Entries are + * keyed by job id and serialized as JSON. Look-ups by message id walk the bucket + * keys; for the volumes rqueue typically tracks (current in-flight + recent retry history) this + * is acceptable for v1 — the Redis impl uses an explicit reverse index, that's a follow-up here. + * + *

The {@code expiry} on save / create is currently best-effort: the bucket is created on the + * first call with the expiry as its TTL. Subsequent saves reuse the same bucket regardless of + * the requested per-key expiry. + */ +@Repository +@Conditional(NatsBackendCondition.class) +@DependsOn("natsKvBucketValidator") +public class NatsRqueueJobDao implements RqueueJobDao { + + private static final Logger log = Logger.getLogger(NatsRqueueJobDao.class.getName()); + private static final String BUCKET_NAME = NatsKvBuckets.JOBS; + + private final NatsProvisioner provisioner; + private final com.github.sonus21.rqueue.serdes.RqueueSerDes serdes; + + public NatsRqueueJobDao( + NatsProvisioner provisioner, com.github.sonus21.rqueue.serdes.RqueueSerDes serdes) { + this.provisioner = provisioner; + this.serdes = serdes; + } + + private KeyValue kv(Duration ttl) throws IOException, JetStreamApiException { + return provisioner.ensureKv(BUCKET_NAME, ttl); + } + + @Override + public void createJob(RqueueJob rqueueJob, Duration expiry) { + save(rqueueJob, expiry); + } + + @Override + public void save(RqueueJob rqueueJob, Duration expiry) { + try { + kv(expiry).put(sanitize(rqueueJob.getId()), serialize(rqueueJob)); + } catch (IOException | JetStreamApiException e) { + log.log(Level.WARNING, "save job " + rqueueJob.getId() + " failed", e); + } + } + + @Override + public RqueueJob findById(String jobId) { + return loadByKey(sanitize(jobId)); + } + + @Override + public List findJobsByIdIn(Collection jobIds) { + List out = new ArrayList<>(jobIds.size()); + for (String id : jobIds) { + RqueueJob j = findById(id); + if (j != null) { + out.add(j); + } + } + return out; + } + + @Override + public List finByMessageId(String messageId) { + if (messageId == null) { + return Collections.emptyList(); + } + return scanForMessageIds(Collections.singletonList(messageId)); + } + + @Override + public List finByMessageIdIn(List messageIds) { + if (messageIds == null || messageIds.isEmpty()) { + return Collections.emptyList(); + } + return scanForMessageIds(messageIds); + } + + @Override + public void delete(String jobId) { + try { + kv(null).delete(sanitize(jobId)); + } catch (IOException | JetStreamApiException e) { + log.log(Level.WARNING, "delete job " + jobId + " failed", e); + } + } + + // ---- helpers ---------------------------------------------------------- + + private RqueueJob loadByKey(String key) { + try { + KeyValueEntry entry = kv(null).get(key); + if (entry == null || entry.getValue() == null) { + return null; + } + return deserialize(entry.getValue()); + } catch (IOException | JetStreamApiException e) { + log.log(Level.WARNING, "loadByKey " + key + " failed", e); + return null; + } + } + + private List scanForMessageIds(Collection messageIds) { + try { + List keys = new ArrayList<>(kv(null).keys()); + List out = new ArrayList<>(); + for (String k : keys) { + RqueueJob j = loadByKey(k); + if (j != null && messageIds.contains(j.getMessageId())) { + out.add(j); + } + } + return out; + } catch (IOException | JetStreamApiException | InterruptedException e) { + log.log(Level.WARNING, "scanForMessageIds failed", e); + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + return Collections.emptyList(); + } + } + + private byte[] serialize(RqueueJob job) throws IOException { + return serdes.serialize(job); + } + + private RqueueJob deserialize(byte[] bytes) { + try { + return serdes.deserialize(bytes, RqueueJob.class); + } catch (Exception e) { + log.log(Level.WARNING, "deserialize RqueueJob failed", e); + return null; + } + } + + /** KV keys allow {@code [A-Za-z0-9_=.-]} only. */ + private static String sanitize(String key) { + return key == null ? "_" : key.replaceAll("[^A-Za-z0-9_=.-]", "_"); + } +} diff --git a/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/dao/NatsRqueueQStatsDao.java b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/dao/NatsRqueueQStatsDao.java new file mode 100644 index 000000000..141540125 --- /dev/null +++ b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/dao/NatsRqueueQStatsDao.java @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + */ + +package com.github.sonus21.rqueue.nats.dao; + +import com.github.sonus21.rqueue.config.NatsBackendCondition; +import com.github.sonus21.rqueue.dao.RqueueQStatsDao; +import com.github.sonus21.rqueue.models.db.QueueStatistics; +import com.github.sonus21.rqueue.nats.internal.NatsProvisioner; +import com.github.sonus21.rqueue.nats.kv.NatsKvBuckets; +import io.nats.client.JetStreamApiException; +import io.nats.client.KeyValue; +import io.nats.client.api.KeyValueEntry; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.DependsOn; +import org.springframework.stereotype.Repository; + +/** + * NATS-backed {@link RqueueQStatsDao} using the {@code rqueue-queue-stats} JetStream KV bucket. + * + *

Each {@link QueueStatistics} entry is stored as a single KV record keyed by its + * {@link QueueStatistics#getId()} (e.g. {@code __rq::q-stat::job-morgue}), serialized via Java + * serialization to match the other NATS DAO implementations. The key is sanitized to the NATS KV + * character set before storage. + * + *

No bucket-level TTL is set: the aggregator service calls + * {@link QueueStatistics#pruneStats} before each {@link #save}, so stale per-day entries inside + * the object are trimmed to {@code rqueue.web.statistic.history.day} days automatically. + */ +@Repository +@Conditional(NatsBackendCondition.class) +@DependsOn("natsKvBucketValidator") +public class NatsRqueueQStatsDao implements RqueueQStatsDao { + + private static final Logger log = Logger.getLogger(NatsRqueueQStatsDao.class.getName()); + private static final String BUCKET_NAME = NatsKvBuckets.QUEUE_STATS; + + private final NatsProvisioner provisioner; + private final com.github.sonus21.rqueue.serdes.RqueueSerDes serdes; + + public NatsRqueueQStatsDao( + NatsProvisioner provisioner, com.github.sonus21.rqueue.serdes.RqueueSerDes serdes) { + this.provisioner = provisioner; + this.serdes = serdes; + } + + private KeyValue kv() throws IOException, JetStreamApiException { + return provisioner.ensureKv(BUCKET_NAME, null); + } + + @Override + public QueueStatistics findById(String id) { + if (id == null) { + return null; + } + try { + KeyValueEntry entry = kv().get(sanitize(id)); + if (entry == null || entry.getValue() == null) { + return null; + } + return deserialize(entry.getValue()); + } catch (IOException | JetStreamApiException e) { + log.log(Level.WARNING, "findById id=" + id + " failed", e); + return null; + } + } + + @Override + public List findAll(Collection ids) { + List out = new ArrayList<>(ids.size()); + for (String id : ids) { + QueueStatistics stat = findById(id); + if (stat != null) { + out.add(stat); + } + } + return out; + } + + @Override + public void save(QueueStatistics queueStatistics) { + if (queueStatistics == null) { + throw new IllegalArgumentException("queueStatistics cannot be null"); + } + if (queueStatistics.getId() == null) { + throw new IllegalArgumentException("id cannot be null: " + queueStatistics); + } + try { + kv().put(sanitize(queueStatistics.getId()), serialize(queueStatistics)); + } catch (IOException | JetStreamApiException e) { + log.log(Level.WARNING, "save id=" + queueStatistics.getId() + " failed", e); + } + } + + // ---- helpers ---------------------------------------------------------- + + private byte[] serialize(QueueStatistics stat) throws IOException { + return serdes.serialize(stat); + } + + private QueueStatistics deserialize(byte[] bytes) { + try { + return serdes.deserialize(bytes, QueueStatistics.class); + } catch (Exception e) { + log.log(Level.WARNING, "deserialize QueueStatistics failed", e); + return null; + } + } + + /** KV keys allow {@code [A-Za-z0-9_=.-]} only. */ + private static String sanitize(String key) { + return key == null ? "_" : key.replaceAll("[^A-Za-z0-9_=.-]", "_"); + } +} diff --git a/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/dao/NatsRqueueSystemConfigDao.java b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/dao/NatsRqueueSystemConfigDao.java new file mode 100644 index 000000000..d48e9a43f --- /dev/null +++ b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/dao/NatsRqueueSystemConfigDao.java @@ -0,0 +1,196 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ + +package com.github.sonus21.rqueue.nats.dao; + +import com.github.sonus21.rqueue.config.NatsBackendCondition; +import com.github.sonus21.rqueue.dao.RqueueSystemConfigDao; +import com.github.sonus21.rqueue.models.db.QueueConfig; +import com.github.sonus21.rqueue.nats.internal.NatsProvisioner; +import com.github.sonus21.rqueue.nats.kv.NatsKvBuckets; +import io.nats.client.JetStreamApiException; +import io.nats.client.KeyValue; +import io.nats.client.api.KeyValueEntry; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.DependsOn; +import org.springframework.stereotype.Repository; + +/** + * NATS-backed {@link RqueueSystemConfigDao} using a JetStream KV bucket as the queue-config + * store. Entries are keyed by {@link QueueConfig#getName()} and serialized via standard Java + * serialization, matching the Redis impl which also relies on + * {@link com.github.sonus21.rqueue.models.SerializableBase}. + * + *

An in-process cache mirrors the Redis impl's {@code byCachedXxx} methods for parity. + * {@link #clearCacheByName(String)} evicts; {@link #saveQConfig(QueueConfig)} keeps the cache + * in sync. + */ +@Repository +@Conditional(NatsBackendCondition.class) +@DependsOn("natsKvBucketValidator") +public class NatsRqueueSystemConfigDao implements RqueueSystemConfigDao { + + private static final Logger log = Logger.getLogger(NatsRqueueSystemConfigDao.class.getName()); + private static final String BUCKET_NAME = NatsKvBuckets.QUEUE_CONFIG; + + private final NatsProvisioner provisioner; + private final com.github.sonus21.rqueue.serdes.RqueueSerDes serdes; + private final ConcurrentHashMap cache = new ConcurrentHashMap<>(); + + public NatsRqueueSystemConfigDao( + NatsProvisioner provisioner, com.github.sonus21.rqueue.serdes.RqueueSerDes serdes) { + this.provisioner = provisioner; + this.serdes = serdes; + } + + private KeyValue kv() throws IOException, JetStreamApiException { + return provisioner.ensureKv(BUCKET_NAME, null); + } + + @Override + public QueueConfig getConfigByName(String name) { + return getConfigByName(name, true); + } + + @Override + public QueueConfig getConfigByName(String name, boolean cached) { + if (cached) { + QueueConfig hit = cache.get(name); + if (hit != null) { + return hit; + } + } + QueueConfig loaded = loadByKey(sanitize(name)); + if (loaded != null) { + cache.put(name, loaded); + } + return loaded; + } + + @Override + public QueueConfig getQConfig(String id, boolean cached) { + if (cached) { + for (QueueConfig hit : cache.values()) { + if (id != null && id.equals(hit.getId())) { + return hit; + } + } + } + return scanForId(id); + } + + @Override + public List getConfigByNames(Collection names) { + List out = new ArrayList<>(names.size()); + for (String n : names) { + QueueConfig c = getConfigByName(n); + if (c != null) { + out.add(c); + } + } + return out; + } + + @Override + public List findAllQConfig(Collection ids) { + List out = new ArrayList<>(ids.size()); + for (String id : ids) { + QueueConfig c = getQConfig(id, true); + if (c != null) { + out.add(c); + } + } + return out; + } + + @Override + public void saveQConfig(QueueConfig queueConfig) { + try { + kv().put(sanitize(queueConfig.getName()), serialize(queueConfig)); + cache.put(queueConfig.getName(), queueConfig); + } catch (IOException | JetStreamApiException e) { + log.log(Level.WARNING, "saveQConfig " + queueConfig.getName() + " failed", e); + } + } + + @Override + public void saveAllQConfig(List newConfigs) { + for (QueueConfig c : newConfigs) { + saveQConfig(c); + } + } + + @Override + public void clearCacheByName(String name) { + cache.remove(name); + } + + // ---- helpers ---------------------------------------------------------- + + private QueueConfig loadByKey(String key) { + try { + KeyValueEntry entry = kv().get(key); + if (entry == null || entry.getValue() == null) { + return null; + } + return deserialize(entry.getValue()); + } catch (IOException | JetStreamApiException e) { + log.log(Level.WARNING, "loadByKey " + key + " failed", e); + return null; + } + } + + private QueueConfig scanForId(String id) { + if (id == null) { + return null; + } + try { + List keys = new ArrayList<>(kv().keys()); + for (String k : keys) { + QueueConfig c = loadByKey(k); + if (c != null && id.equals(c.getId())) { + return c; + } + } + return null; + } catch (IOException | JetStreamApiException | InterruptedException e) { + log.log(Level.WARNING, "scanForId " + id + " failed", e); + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + return null; + } + } + + private byte[] serialize(QueueConfig c) throws IOException { + return serdes.serialize(c); + } + + private QueueConfig deserialize(byte[] bytes) { + try { + return serdes.deserialize(bytes, QueueConfig.class); + } catch (Exception e) { + log.log(Level.WARNING, "deserialize QueueConfig failed", e); + return null; + } + } + + /** KV keys allow {@code [A-Za-z0-9_=.-]} only. */ + private static String sanitize(String key) { + return key == null ? "_" : key.replaceAll("[^A-Za-z0-9_=.-]", "_"); + } +} diff --git a/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/internal/NatsProvisioner.java b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/internal/NatsProvisioner.java new file mode 100644 index 000000000..f6cf1b5a0 --- /dev/null +++ b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/internal/NatsProvisioner.java @@ -0,0 +1,333 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ +package com.github.sonus21.rqueue.nats.internal; + +import com.github.sonus21.rqueue.enums.QueueType; +import com.github.sonus21.rqueue.nats.RqueueNatsConfig; +import com.github.sonus21.rqueue.nats.RqueueNatsException; +import io.nats.client.Connection; +import io.nats.client.JetStreamApiException; +import io.nats.client.JetStreamManagement; +import io.nats.client.KeyValue; +import io.nats.client.KeyValueManagement; +import io.nats.client.api.AckPolicy; +import io.nats.client.api.CompressionOption; +import io.nats.client.api.ConsumerConfiguration; +import io.nats.client.api.ConsumerInfo; +import io.nats.client.api.DeliverPolicy; +import io.nats.client.api.KeyValueConfiguration; +import io.nats.client.api.KeyValueStatus; +import io.nats.client.api.RetentionPolicy; +import io.nats.client.api.StreamConfiguration; +import io.nats.client.api.StreamInfo; +import java.io.IOException; +import java.time.Duration; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Idempotent stream/consumer/KV provisioning helpers. All methods are safe to call repeatedly; + * if the target object already exists with a compatible config, this class leaves it alone. If it + * exists but with diverging config (e.g. ackWait, maxDeliver), this class logs a WARN and does + * NOT auto-mutate so user customizations are preserved. + */ +public class NatsProvisioner { + + private static final Logger log = Logger.getLogger(NatsProvisioner.class.getName()); + + private final Connection connection; + private final KeyValueManagement kvm; + private final JetStreamManagement jsm; + private final RqueueNatsConfig config; + + private final ConcurrentHashMap kvCache = new ConcurrentHashMap<>(); + private final ConcurrentHashMap kvLocks = new ConcurrentHashMap<>(); + + // stream name → provisioned (set membership acts as the boolean flag) + private final Set streamsDone = ConcurrentHashMap.newKeySet(); + private final ConcurrentHashMap streamLocks = new ConcurrentHashMap<>(); + + // "streamName/requestedConsumerName" → actual consumer name (may differ for stale-rebind) + private final ConcurrentHashMap consumerCache = new ConcurrentHashMap<>(); + private final ConcurrentHashMap consumerLocks = new ConcurrentHashMap<>(); + + public NatsProvisioner(Connection connection, JetStreamManagement jsm, RqueueNatsConfig config) + throws IOException { + this.connection = connection; + this.kvm = connection.keyValueManagement(); + this.jsm = jsm; + this.config = config; + } + + // ---- KV provisioning -------------------------------------------------- + + /** + * Returns a {@link KeyValue} handle for {@code bucketName}, creating the bucket on first call. + * All buckets are created with S2 compression. A bucket-level TTL is applied at creation time + * only; existing buckets are reused as-is. + * + * @param bucketName NATS KV bucket name (e.g. {@code "rqueue-jobs"}) + * @param ttl bucket-level max-age; {@code null} or non-positive = no TTL + */ + public KeyValue ensureKv(String bucketName, Duration ttl) + throws IOException, JetStreamApiException { + KeyValue cached = kvCache.get(bucketName); + if (cached != null) { + return cached; + } + Object lock = kvLocks.computeIfAbsent(bucketName, k -> new Object()); + synchronized (lock) { + cached = kvCache.get(bucketName); + if (cached != null) { + return cached; + } + try { + KeyValueStatus status = kvm.getStatus(bucketName); + if (status != null) { + KeyValue kv = connection.keyValue(bucketName); + kvCache.put(bucketName, kv); + return kv; + } + } catch (JetStreamApiException missing) { + // bucket absent — fall through to create + } + RqueueNatsConfig.StreamDefaults sd = config.getStreamDefaults(); + KeyValueConfiguration.Builder cfg = KeyValueConfiguration.builder() + .name(bucketName) + .compression(true) + .replicas(sd.getReplicas()) + .storageType(sd.getStorage()); + if (ttl != null && !ttl.isZero() && !ttl.isNegative()) { + cfg.ttl(ttl); + } + kvm.create(cfg.build()); + KeyValue kv = connection.keyValue(bucketName); + kvCache.put(bucketName, kv); + return kv; + } + } + + // ---- Stream provisioning ---------------------------------------------- + + /** + * Ensure a JetStream stream exists with the given subjects, using {@link QueueType#QUEUE} + * (WorkQueue retention) as the default. Callers that have a {@link QueueType} available should + * use {@link #ensureStream(String, List, QueueType)} instead. + */ + public void ensureStream(String streamName, List subjects) { + ensureStream(streamName, subjects, QueueType.QUEUE, null); + } + + /** See {@link #ensureStream(String, List, QueueType, String)}. */ + public void ensureStream(String streamName, List subjects, QueueType queueType) { + ensureStream(streamName, subjects, queueType, null); + } + + /** + * Ensure a JetStream stream exists with the given subjects and retention policy derived from + * {@code queueType}: + *

    + *
  • {@link QueueType#QUEUE} — {@link io.nats.client.api.RetentionPolicy#WorkQueue}: each + * message is delivered to exactly one consumer; competing-consumer semantics. + *
  • {@link QueueType#STREAM} — {@link io.nats.client.api.RetentionPolicy#Limits}: every + * independent durable consumer group receives all messages; stream/fan-out semantics. + *
+ * + *

{@code description} is forwarded to JetStream as the stream's description (visible via + * {@code nats stream info}). Callers should pass the rqueue queue name so operators can map a + * stream back to the queue that created it; pass {@code null} to skip. + * + *

Hits the NATS backend at most once per stream name per process lifetime; subsequent calls + * return immediately from the in-process cache. If the stream already exists with a different + * retention policy, a WARNING is logged and the existing config is left untouched. + */ + public void ensureStream( + String streamName, List subjects, QueueType queueType, String description) { + if (streamsDone.contains(streamName)) { + return; + } + Object lock = streamLocks.computeIfAbsent(streamName, k -> new Object()); + synchronized (lock) { + if (streamsDone.contains(streamName)) { + return; + } + try { + StreamInfo existing = safeGetStreamInfo(streamName); + RetentionPolicy desired = + queueType == QueueType.STREAM ? RetentionPolicy.Limits : RetentionPolicy.WorkQueue; + if (existing == null) { + if (!config.isAutoCreateStreams()) { + throw new RqueueNatsException( + "Stream '" + streamName + "' does not exist and autoCreateStreams=false"); + } + RqueueNatsConfig.StreamDefaults sd = config.getStreamDefaults(); + StreamConfiguration.Builder b = StreamConfiguration.builder() + .name(streamName) + .subjects(subjects) + .replicas(sd.getReplicas()) + .storageType(sd.getStorage()) + .retentionPolicy(desired) + .duplicateWindow(sd.getDuplicateWindow()) + .compressionOption(CompressionOption.S2); + if (description != null && !description.isEmpty()) { + b.description(description); + } + if (sd.getMaxMsgs() > 0) { + b.maxMessages(sd.getMaxMsgs()); + } + if (sd.getMaxBytes() > 0) { + b.maxBytes(sd.getMaxBytes()); + } + if (sd.getMaxAge() != null + && !sd.getMaxAge().isZero() + && !sd.getMaxAge().isNegative()) { + b.maxAge(sd.getMaxAge()); + } + jsm.addStream(b.build()); + } else { + RetentionPolicy actual = existing.getConfiguration().getRetentionPolicy(); + if (actual != desired) { + log.log( + Level.WARNING, + "Stream ''{0}'' exists with retention={1} but queueMode requires retention={2}" + + " — leaving existing config in place.", + new Object[] {streamName, actual, desired}); + } + } + } catch (IOException | JetStreamApiException e) { + throw new RqueueNatsException( + "Failed to ensure stream '" + streamName + "' for subjects " + subjects, e); + } + streamsDone.add(streamName); + } + } + + /** + * Ensure a durable pull consumer exists, returning the consumer name. + * Hits the NATS backend at most once per (stream, consumer) pair per process lifetime. + * + *

No filter subject is set on the consumer: each queue already has its own dedicated stream + * with a single subject, so a filter would be redundant. More importantly, omitting the filter + * allows multiple independent consumer groups (fan-out) to coexist on the same stream — NATS + * rejects two consumers with the same filter subject (error 10100) regardless of retention type. + */ + public String ensureConsumer( + String streamName, + String consumerName, + Duration ackWait, + long maxDeliver, + long maxAckPending) { + String cacheKey = streamName + "/" + consumerName; + String cached = consumerCache.get(cacheKey); + if (cached != null) { + return cached; + } + Object lock = consumerLocks.computeIfAbsent(cacheKey, k -> new Object()); + synchronized (lock) { + cached = consumerCache.get(cacheKey); + if (cached != null) { + return cached; + } + String actual = + doEnsureConsumer(streamName, consumerName, ackWait, maxDeliver, maxAckPending); + consumerCache.put(cacheKey, actual); + return actual; + } + } + + private String doEnsureConsumer( + String streamName, + String consumerName, + Duration ackWait, + long maxDeliver, + long maxAckPending) { + try { + ConsumerInfo info = safeGetConsumerInfo(streamName, consumerName); + if (info != null) { + ConsumerConfiguration cc = info.getConsumerConfiguration(); + if (cc.getAckWait() != null && !cc.getAckWait().equals(ackWait)) { + log.log( + Level.WARNING, + "Consumer " + streamName + "/" + consumerName + + " ackWait differs (existing=" + cc.getAckWait() + + ", desired=" + ackWait + ") - leaving existing config in place."); + } + if (cc.getMaxDeliver() != maxDeliver) { + log.log( + Level.WARNING, + "Consumer " + streamName + "/" + consumerName + + " maxDeliver differs (existing=" + cc.getMaxDeliver() + + ", desired=" + maxDeliver + ") - leaving existing config in place."); + } + return consumerName; + } + if (!config.isAutoCreateConsumers()) { + throw new RqueueNatsException("Consumer '" + consumerName + "' on stream '" + streamName + + "' does not exist and autoCreateConsumers=false"); + } + jsm.addOrUpdateConsumer( + streamName, + ConsumerConfiguration.builder() + .durable(consumerName) + .ackPolicy(AckPolicy.Explicit) + .deliverPolicy(DeliverPolicy.All) + .ackWait(ackWait) + .maxDeliver(maxDeliver) + .maxAckPending(maxAckPending) + .build()); + return consumerName; + } catch (JetStreamApiException e) { + throw new RqueueNatsException( + "Failed to ensure consumer '" + consumerName + "' on stream '" + streamName + "'", e); + } catch (IOException e) { + throw new RqueueNatsException( + "Failed to ensure consumer '" + consumerName + "' on stream '" + streamName + "'", e); + } + } + + /** Ensure a DLQ stream exists capturing dead-letter subjects (e.g. "rqueue.js.*.dlq"). */ + public void ensureDlqStream(String dlqStreamName, List dlqSubjects) { + if (!config.isAutoCreateDlqStream()) { + return; + } + ensureStream(dlqStreamName, dlqSubjects); + } + + // ---- private helpers -------------------------------------------------- + + private StreamInfo safeGetStreamInfo(String streamName) + throws IOException, JetStreamApiException { + try { + return jsm.getStreamInfo(streamName); + } catch (JetStreamApiException e) { + // 10059 = stream not found + if (e.getApiErrorCode() == 10059 || e.getErrorCode() == 404) { + return null; + } + throw e; + } + } + + private ConsumerInfo safeGetConsumerInfo(String streamName, String consumerName) + throws IOException, JetStreamApiException { + try { + return jsm.getConsumerInfo(streamName, consumerName); + } catch (JetStreamApiException e) { + // 10014 = consumer not found, 10059 = stream not found + if (e.getApiErrorCode() == 10014 || e.getApiErrorCode() == 10059 || e.getErrorCode() == 404) { + return null; + } + throw e; + } + } +} diff --git a/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/js/JetStreamMessageBroker.java b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/js/JetStreamMessageBroker.java new file mode 100644 index 000000000..3e64ca0a3 --- /dev/null +++ b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/js/JetStreamMessageBroker.java @@ -0,0 +1,788 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ +package com.github.sonus21.rqueue.nats.js; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.github.sonus21.rqueue.core.RqueueMessage; +import com.github.sonus21.rqueue.core.spi.Capabilities; +import com.github.sonus21.rqueue.core.spi.MessageBroker; +import com.github.sonus21.rqueue.enums.QueueType; +import com.github.sonus21.rqueue.listener.QueueDetail; +import com.github.sonus21.rqueue.nats.RqueueNatsConfig; +import com.github.sonus21.rqueue.nats.RqueueNatsException; +import com.github.sonus21.rqueue.nats.internal.NatsProvisioner; +import com.github.sonus21.rqueue.serdes.RqJacksonSerDes; +import com.github.sonus21.rqueue.serdes.RqueueSerDes; +import com.github.sonus21.rqueue.serdes.SerializationUtils; +import com.github.sonus21.rqueue.utils.PriorityUtils; +import io.nats.client.Connection; +import io.nats.client.Dispatcher; +import io.nats.client.JetStream; +import io.nats.client.JetStreamApiException; +import io.nats.client.JetStreamManagement; +import io.nats.client.JetStreamSubscription; +import io.nats.client.Message; +import io.nats.client.PullSubscribeOptions; +import io.nats.client.api.AckPolicy; +import io.nats.client.api.ConsumerConfiguration; +import io.nats.client.api.DeliverPolicy; +import io.nats.client.impl.Headers; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Consumer; +import java.util.logging.Level; +import java.util.logging.Logger; +import reactor.core.publisher.Mono; + +/** + * JetStream-backed implementation of {@link MessageBroker}. + * + *

This class keeps a per-instance in-memory map ({@code inFlight}) of NATS messages popped via + * {@link #pop} so that {@link #ack} / {@link #nack} can locate the underlying NATS message handle. + * The map is intentionally local: a process restart loses any pending entries, which is consistent + * with the v1 capability set declaring no scheduled introspection. NATS itself will redeliver + * unacked messages after {@code ackWait}. + * + *

Delayed enqueue and any scheduled/cron features throw {@link UnsupportedOperationException}. + * {@code moveExpired} is a no-op returning 0; redelivery is handled by JetStream's ack-wait timer. + */ +public class JetStreamMessageBroker implements MessageBroker, AutoCloseable { + + private static final Logger log = Logger.getLogger(JetStreamMessageBroker.class.getName()); + private static final Capabilities CAPS = new Capabilities(false, false, false, false); + + /** + * Lower bound for fetch wait when the caller passes a non-positive duration. JetStream rejects + * zero on a pull fetch, so any zero/negative wait is rounded up to this minimum. Callers that + * want long-poll semantics should pass the desired wait explicitly (e.g. the listener + * container's {@code pollingInterval}); this constant only guards against accidental zero waits + * from non-listener callers. + */ + private static final Duration MIN_FETCH_WAIT = Duration.ofMillis(50); + + private final Connection connection; + private final JetStream js; + private final JetStreamManagement jsm; + private final RqueueNatsConfig config; + private final RqueueSerDes serdes; + private final NatsProvisioner provisioner; + + /** + * keyed by RqueueMessage.id, value is the underlying NATS Message for ack/nak. + */ + private final ConcurrentHashMap inFlight = new ConcurrentHashMap<>(); + + /** + * Cached pull subscriptions keyed by stream + consumerName so we don't re-bind on every pop. + */ + private final ConcurrentHashMap subscriptionCache = + new ConcurrentHashMap<>(); + + // Public so tests in sibling packages (e.g. JetStreamMessageBrokerDelayThrowsTest) can build a + // broker directly without going through builder() — keeps the regression caught by that test + // pinned to the constructor signature itself. + public JetStreamMessageBroker( + Connection connection, + JetStream js, + JetStreamManagement jsm, + RqueueNatsConfig config, + RqueueSerDes serdes, + NatsProvisioner provisioner) { + this.connection = connection; + this.js = js; + this.jsm = jsm; + this.config = config; + this.serdes = serdes; + this.provisioner = provisioner; + } + + public static Builder builder() { + return new Builder(); + } + + // ---- subject / stream naming ------------------------------------------- + + private String subjectFor(QueueDetail q) { + return config.getSubjectPrefix() + q.getName(); + } + + private String streamFor(QueueDetail q) { + return config.getStreamPrefix() + q.getName(); + } + + /** + * Resolve the priority-specific subject. Uses the same {@code "_priority"} suffix as + * {@link com.github.sonus21.rqueue.utils.PriorityUtils#getSuffix(String)} so the subject + * matches the expanded {@link QueueDetail#getName()} used by the poller (e.g. {@code "pq_high"}). + */ + private String subjectFor(QueueDetail q, String priority) { + if (priority == null || priority.isEmpty()) { + return subjectFor(q); + } + return config.getSubjectPrefix() + q.getName() + PriorityUtils.getSuffix(priority); + } + + /** + * Resolve the priority-specific stream. Uses the same {@code "_priority"} suffix as + * {@link com.github.sonus21.rqueue.utils.PriorityUtils#getSuffix(String)} so the stream name + * matches what the poller derives from the expanded {@link QueueDetail#getName()}. + */ + private String streamFor(QueueDetail q, String priority) { + if (priority == null || priority.isEmpty()) { + return streamFor(q); + } + return config.getStreamPrefix() + q.getName() + PriorityUtils.getSuffix(priority); + } + + private String dlqStreamFor(QueueDetail q) { + return streamFor(q) + config.getDlqStreamSuffix(); + } + + private String dlqSubjectFor(QueueDetail q) { + return subjectFor(q) + config.getDlqSubjectSuffix(); + } + + /** Stream description shown in {@code nats stream info} so operators can map back to rqueue. */ + private static String streamDescription(QueueDetail q) { + return "rqueue queue: " + q.getName(); + } + + /** Stream description for the priority sub-stream. */ + private static String streamDescription(QueueDetail q, String priority) { + return priority == null || priority.isEmpty() + ? streamDescription(q) + : "rqueue queue: " + q.getName() + " (priority=" + priority + ")"; + } + + private static String dlqStreamDescription(QueueDetail q) { + return "rqueue DLQ for queue: " + q.getName(); + } + + // ---- MessageBroker ----------------------------------------------------- + + @Override + public void enqueue(QueueDetail q, RqueueMessage m) { + String subject = subjectFor(q); + String stream = streamFor(q); + provisioner.ensureStream(stream, List.of(subject), q.getType(), streamDescription(q)); + Headers headers = new Headers(); + if (m.getId() != null) { + headers.add("Nats-Msg-Id", m.getId()); + } + try { + byte[] payload = serdes.serialize(m); + js.publish(subject, headers, payload); + } catch (IOException | JetStreamApiException e) { + throw new RqueueNatsException( + "Failed to enqueue message id=" + + m.getId() + + " queue=" + + q.getName() + + " subject=" + + subject, + e); + } catch (RuntimeException e) { + throw new RqueueNatsException( + "Failed to serialize/enqueue message id=" + + m.getId() + + " queue=" + + q.getName() + + " subject=" + + subject, + e); + } + } + + @Override + public void enqueue(QueueDetail q, String priority, RqueueMessage m) { + String subject = subjectFor(q, priority); + String stream = streamFor(q, priority); + provisioner.ensureStream(stream, List.of(subject), q.getType(), streamDescription(q, priority)); + Headers headers = new Headers(); + if (m.getId() != null) { + headers.add("Nats-Msg-Id", m.getId()); + } + try { + byte[] payload = serdes.serialize(m); + js.publish(subject, headers, payload); + } catch (IOException | JetStreamApiException e) { + throw new RqueueNatsException( + "Failed to enqueue message id=" + + m.getId() + + " queue=" + + q.getName() + + " priority=" + + priority + + " subject=" + + subject, + e); + } catch (RuntimeException e) { + throw new RqueueNatsException( + "Failed to serialize/enqueue message id=" + + m.getId() + + " queue=" + + q.getName() + + " priority=" + + priority + + " subject=" + + subject, + e); + } + } + + @Override + public void enqueueWithDelay(QueueDetail q, RqueueMessage m, long delayMs) { + throw new UnsupportedOperationException( + "delayed enqueue not supported by NATS backend in this version; " + + "use the Redis backend for scheduled messages"); + } + + @Override + public Mono enqueueReactive(QueueDetail q, RqueueMessage m) { + String subject = subjectFor(q); + String stream = streamFor(q); + try { + provisioner.ensureStream(stream, List.of(subject), q.getType(), streamDescription(q)); + } catch (Exception e) { + return Mono.error(new RqueueNatsException( + "Failed to provision stream for reactive enqueue id=" + + m.getId() + + " queue=" + + q.getName(), + e)); + } + Headers headers = new Headers(); + if (m.getId() != null) { + headers.add("Nats-Msg-Id", m.getId()); + } + byte[] payload; + try { + payload = serdes.serialize(m); + } catch (RuntimeException | IOException e) { + return Mono.error(new RqueueNatsException( + "Failed to serialize message id=" + + m.getId() + + " queue=" + + q.getName() + + " subject=" + + subject, + e)); + } + return Mono.fromFuture(() -> js.publishAsync(subject, headers, payload)) + .onErrorMap(e -> e instanceof RqueueNatsException + ? e + : new RqueueNatsException( + "Failed to enqueue message id=" + + m.getId() + + " queue=" + + q.getName() + + " subject=" + + subject, + e)) + .then(); + } + + @Override + public Mono enqueueWithDelayReactive(QueueDetail q, RqueueMessage m, long delayMs) { + return Mono.error(new UnsupportedOperationException( + "delayed enqueue not supported by NATS backend in this version; " + + "use the Redis backend for scheduled messages")); + } + + @Override + public List pop(QueueDetail q, String consumerName, int batch, Duration wait) { + return popInternal( + streamFor(q), + subjectFor(q), + resolveConsumerName(q.getName(), consumerName), + batch, + wait, + resolveAckWait(q, config), + resolveMaxDeliver(q, config)); + } + + @Override + public List pop( + QueueDetail q, String priority, String consumerName, int batch, Duration wait) { + return popInternal( + streamFor(q, priority), + subjectFor(q, priority), + resolveConsumerName(q.getName(), consumerName), + batch, + wait, + resolveAckWait(q, config), + resolveMaxDeliver(q, config)); + } + + private static String resolveConsumerName(String queueName, String consumerName) { + return (consumerName != null && !consumerName.isEmpty()) ? consumerName : "rqueue-" + queueName; + } + + /** + * Resolve the JetStream {@code ackWait} for this queue's pull consumer: per-queue + * {@link QueueDetail#getVisibilityTimeout()} (when positive), else the global + * {@code RqueueNatsConfig.ConsumerDefaults.getAckWait()}. Honouring visibilityTimeout makes + * the NATS backend match the contract every other rqueue backend exposes: a message stays + * invisible to other consumers for that window and is redelivered if not acked in time. + */ + public static Duration resolveAckWait(QueueDetail q, RqueueNatsConfig config) { + long vt = q.getVisibilityTimeout(); + if (vt > 0) { + return Duration.ofMillis(vt); + } + return config.getConsumerDefaults().getAckWait(); + } + + /** + * Resolve the JetStream {@code maxDeliver} from per-queue {@link QueueDetail#getNumRetry()} + * (counted as initial delivery + N retries = numRetry + 1). The {@link Integer#MAX_VALUE} + * "retry forever" sentinel maps to JetStream's unlimited value ({@code -1}); non-positive + * numRetry falls back to {@code RqueueNatsConfig.ConsumerDefaults.getMaxDeliver()}. + */ + public static long resolveMaxDeliver(QueueDetail q, RqueueNatsConfig config) { + int numRetry = q.getNumRetry(); + if (numRetry == Integer.MAX_VALUE) { + return -1L; + } + if (numRetry > 0) { + return numRetry + 1L; + } + return config.getConsumerDefaults().getMaxDeliver(); + } + + private List popInternal( + String stream, + String subject, + String consumerName, + int batch, + Duration wait, + Duration ackWait, + long maxDeliver) { + // Honour the caller-supplied wait — this is the listener container's pollingInterval for + // RqueueMessagePoller, and lets JetStream long-poll instead of the broker firing a steady + // stream of $JS.API.CONSUMER.MSG.NEXT requests. Only fall back when the caller didn't + // express a preference; zero/negative waits are rounded up to the JetStream minimum. + Duration fetchWait; + if (wait == null) { + fetchWait = config.getDefaultFetchWait(); + } else if (wait.isZero() || wait.isNegative()) { + fetchWait = MIN_FETCH_WAIT; + } else { + fetchWait = wait; + } + String key = stream + "/" + consumerName; + JetStreamSubscription sub = subscriptionCache.computeIfAbsent(key, k -> { + // NatsStreamValidator provisions the stream and consumer at bootstrap (RqueueBootstrapEvent). + // NatsProvisioner caches both, so ensureConsumer here is a map lookup — no backend call. + try { + String actualConsumerName = provisioner.ensureConsumer( + stream, + consumerName, + ackWait, + maxDeliver, + config.getConsumerDefaults().getMaxAckPending()); + PullSubscribeOptions opts = PullSubscribeOptions.bind(stream, actualConsumerName); + // Consumer has no filter subject; pass null so the NATS client doesn't validate + // the subject against a (nonexistent) filter — SUB-90011 otherwise. + return js.subscribe(null, opts); + } catch (IOException | JetStreamApiException e) { + throw new RqueueNatsException( + "Failed to bind pull subscription stream=" + stream + " consumer=" + consumerName, e); + } + }); + + List msgs = sub.fetch(batch, fetchWait); + List out = new ArrayList<>(msgs.size()); + for (Message nm : msgs) { + try { + RqueueMessage rm = serdes.deserialize(nm.getData(), RqueueMessage.class); + // derive failure count from JetStream redelivery metadata + try { + long deliveredCount = nm.metaData().deliveredCount(); + rm.setFailureCount((int) Math.max(0, deliveredCount - 1)); + } catch (Exception ignored) { + // defensive: metadata unavailable on non-JetStream messages + } + if (rm.getId() != null) { + inFlight.put(rm.getId(), nm); + } + out.add(rm); + } catch (RuntimeException | IOException e) { + log.log( + Level.WARNING, + "Failed to deserialize JetStream payload on subject " + + subject + + "; nak'ing for redelivery", + e); + try { + nm.nak(); + } catch (RuntimeException ignored) { + // best-effort + } + } + } + return out; + } + + @Override + public boolean ack(QueueDetail q, RqueueMessage m) { + if (m.getId() == null) { + return false; + } + Message nm = inFlight.remove(m.getId()); + if (nm == null) { + return false; + } + nm.ack(); + return true; + } + + @Override + public boolean nack(QueueDetail q, RqueueMessage m, long retryDelayMs) { + if (m.getId() == null) { + return false; + } + Message nm = inFlight.remove(m.getId()); + if (nm == null) { + return false; + } + nm.nakWithDelay(Duration.ofMillis(Math.max(0L, retryDelayMs))); + return true; + } + + @Override + public void moveToDlq( + QueueDetail source, + String targetQueue, + RqueueMessage old, + RqueueMessage updated, + long delayMs) { + // Ack the original NATS message so it is removed from the source stream. + if (old.getId() != null) { + Message nm = inFlight.remove(old.getId()); + if (nm != null) { + nm.ack(); + } + } + // targetQueue is the configured deadLetterQueue name (e.g. "job-morgue"). Map it to a NATS + // stream and subject using the same prefix convention as any other queue. + // NATS JetStream has no server-side delayed publish, so delayMs is ignored. + String dlqStream = config.getStreamPrefix() + targetQueue; + String dlqSubject = config.getSubjectPrefix() + targetQueue; + Headers headers = new Headers(); + if (updated.getId() != null) { + headers.add("Nats-Msg-Id", updated.getId() + "-dlq"); + } + try { + provisioner.ensureStream( + dlqStream, List.of(dlqSubject), QueueType.QUEUE, "rqueue DLQ for queue: " + targetQueue); + byte[] payload = serdes.serialize(updated); + js.publish(dlqSubject, headers, payload); + } catch (IOException | JetStreamApiException e) { + throw new RqueueNatsException( + "Failed to move message id=" + old.getId() + " to DLQ stream=" + dlqStream, e); + } catch (RuntimeException e) { + throw new RqueueNatsException( + "Failed to serialize/publish message id=" + old.getId() + " to DLQ stream=" + dlqStream, + e); + } + } + + @Override + public long moveExpired(QueueDetail q, long now, int batch) { + // No-op: JetStream's ack-wait + maxDeliver + DLQ advisory bridge handle redelivery and + // dead-lettering. v1 capabilities advertise no scheduled introspection. + return 0L; + } + + @Override + public List peek(QueueDetail q, long offset, long count) { + String stream = streamFor(q); + String subject = subjectFor(q); + JetStreamSubscription sub = null; + try { + ConsumerConfiguration.Builder cb = ConsumerConfiguration.builder() + .ackPolicy(AckPolicy.None) + .filterSubject(subject) + .name("rqueue-js-peek-" + UUID.randomUUID()); + if (offset > 0) { + cb.deliverPolicy(DeliverPolicy.ByStartSequence).startSequence(Math.max(1L, offset)); + } else { + cb.deliverPolicy(DeliverPolicy.All); + } + PullSubscribeOptions opts = PullSubscribeOptions.builder().stream(stream) + .configuration(cb.build()) + .build(); + sub = js.subscribe(subject, opts); + int n = (int) Math.min(Integer.MAX_VALUE, Math.max(0L, count)); + List msgs = sub.fetch(n, Duration.ofSeconds(2)); + List out = new ArrayList<>(msgs.size()); + for (Message nm : msgs) { + try { + out.add(serdes.deserialize(nm.getData(), RqueueMessage.class)); + } catch (Exception e) { + log.log(Level.WARNING, "peek: skipping undeserializable message", e); + } + } + return out; + } catch (IOException | JetStreamApiException e) { + throw new RqueueNatsException( + "Failed to peek queue=" + q.getName() + " offset=" + offset + " count=" + count, e); + } finally { + if (sub != null) { + try { + sub.unsubscribe(); + } catch (RuntimeException ignored) { + // ephemeral consumer is auto-reaped server-side; ignore + } + } + } + } + + @Override + public long size(QueueDetail q) { + String stream = streamFor(q); + try { + return jsm.getStreamInfo(stream).getStreamState().getMsgCount(); + } catch (IOException | JetStreamApiException e) { + throw new RqueueNatsException("Failed to read stream size for queue=" + q.getName(), e); + } + } + + @Override + public AutoCloseable subscribe(String channel, Consumer handler) { + Dispatcher d = + connection.createDispatcher(msg -> handler.accept(new String(msg.getData(), UTF_8))); + d.subscribe(channel); + return () -> { + try { + connection.closeDispatcher(d); + } catch (RuntimeException e) { + // best-effort close + } + }; + } + + @Override + public void publish(String channel, String payload) { + connection.publish(channel, payload.getBytes(UTF_8)); + } + + @Override + public void onQueueRegistered(QueueDetail q) { + String stream = streamFor(q); + String subject = subjectFor(q); + provisioner.ensureStream(stream, List.of(subject), q.getType(), streamDescription(q)); + } + + /** + * NATS subjects use {@code .} as a hierarchy separator and stream / consumer names disallow it + * outright. A queue name like {@code "orders.us"} would (a) silently turn the publish subject + * into a two-level token tree and (b) make {@code rqueue-js-orders.us} an invalid stream name — + * the JetStream API would reject it with an opaque error at first publish. Reject the name at + * registration so the failure is loud and local. + * + *

{@code *} and {@code >} are also illegal in subject tokens (they're NATS wildcards), and + * whitespace is rejected by the server; check those too while we're here. + */ + @Override + public void validateQueueName(String queueName) { + if (queueName == null || queueName.isEmpty()) { + return; + } + for (int i = 0; i < queueName.length(); i++) { + char c = queueName.charAt(i); + if (c == '.' || c == '*' || c == '>' || Character.isWhitespace(c)) { + throw new IllegalArgumentException("Queue name '" + + queueName + + "' contains illegal character '" + + c + + "' for the NATS backend. Subject hierarchy ('.'), wildcards ('*', '>') and" + + " whitespace are not allowed in queue names — use '-' or '_' instead."); + } + } + } + + @Override + public Capabilities capabilities() { + return CAPS; + } + + @Override + public String storageKicker() { + return "NATS"; + } + + @Override + public String storageDescription() { + return "Underlying NATS JetStream streams for the queues visible on this page."; + } + + @Override + public String storageDisplayName(QueueDetail q) { + return streamFor(q); + } + + @Override + public String dlqStorageDisplayName(QueueDetail q) { + return dlqStreamFor(q); + } + + @Override + public void close() { + for (JetStreamSubscription s : subscriptionCache.values()) { + try { + s.unsubscribe(); + } catch (RuntimeException ignored) { + // ignore + } + } + subscriptionCache.clear(); + inFlight.clear(); + } + + /** + * Provision a DLQ stream for the given queue. Caller wires up an advisory listener (subscribed to + * {@code $JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>}) that republishes the exhausted message + * onto {@link #dlqSubjectFor(QueueDetail)}. v1 leaves the bridge wiring opt-in via + * {@link #installDeadLetterBridge(QueueDetail, String)}. + */ + public void provisionDlq(QueueDetail q) { + // Explicit call — always provision, bypassing the autoCreateDlqStream flag. + // That flag gates automatic provisioning at bootstrap; here the caller is explicitly opting in. + provisioner.ensureStream( + dlqStreamFor(q), List.of(dlqSubjectFor(q)), QueueType.QUEUE, dlqStreamDescription(q)); + } + + /** + * Install a background dispatcher that watches max-deliveries advisories on the queue's stream + * and republishes the offending payload onto the DLQ subject. Returns an {@link AutoCloseable} + * that tears the dispatcher down. Tests rely on this; production code in Phase 3 will call it + * during container start. + */ + public AutoCloseable installDeadLetterBridge(QueueDetail q, String consumerName) { + provisionDlq(q); + String advisorySubject = + "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES." + streamFor(q) + "." + consumerName; + String dlqSubject = dlqSubjectFor(q); + String stream = streamFor(q); + Dispatcher d = connection.createDispatcher(advisoryMsg -> { + try { + @SuppressWarnings("unchecked") + Map adv = serdes.deserialize(advisoryMsg.getData(), Map.class); + Object seqVal = adv.get("stream_seq"); + long streamSeq = seqVal instanceof Number ? ((Number) seqVal).longValue() : -1L; + if (streamSeq <= 0) { + return; + } + io.nats.client.api.MessageInfo mi = jsm.getMessage(stream, streamSeq); + Headers h = new Headers(); + if (mi.getHeaders() != null) { + mi.getHeaders().forEach((k, v) -> h.add(k, v)); + } + js.publish(dlqSubject, h, mi.getData()); + } catch (Exception e) { + log.log( + Level.WARNING, "Failed to bridge max-delivery advisory to DLQ for stream=" + stream, e); + } + }); + d.subscribe(advisorySubject); + return () -> { + try { + connection.closeDispatcher(d); + } catch (RuntimeException ignored) { + // best-effort + } + }; + } + + // ---- builder ----------------------------------------------------------- + + public static class Builder { + + private Connection connection; + private JetStream jetStream; + private JetStreamManagement management; + private RqueueNatsConfig config; + private RqueueSerDes serdes; + private NatsProvisioner provisioner; + + public Builder connection(Connection connection) { + this.connection = connection; + return this; + } + + public Builder jetStream(JetStream jetStream) { + this.jetStream = jetStream; + return this; + } + + public Builder management(JetStreamManagement management) { + this.management = management; + return this; + } + + public Builder config(RqueueNatsConfig config) { + this.config = config; + return this; + } + + public Builder serDes(RqueueSerDes serdes) { + this.serdes = serdes; + return this; + } + + public Builder provisioner(NatsProvisioner provisioner) { + this.provisioner = provisioner; + return this; + } + + public JetStreamMessageBroker build() { + if (connection == null) { + throw new IllegalStateException("connection is required"); + } + try { + if (jetStream == null) { + jetStream = connection.jetStream(); + } + if (management == null) { + management = connection.jetStreamManagement(); + } + if (provisioner == null) { + provisioner = new NatsProvisioner( + connection, management, config != null ? config : RqueueNatsConfig.defaults()); + } + } catch (IOException e) { + throw new RqueueNatsException("Failed to derive JetStream context from connection", e); + } + if (config == null) { + config = RqueueNatsConfig.defaults(); + } + if (serdes == null) { + serdes = new RqJacksonSerDes(SerializationUtils.getObjectMapper()); + } + return new JetStreamMessageBroker( + connection, jetStream, management, config, serdes, provisioner); + } + + /** + * Create a broker that wraps a pre-built {@link Map} of NATS handles. Used by the factory. + */ + public JetStreamMessageBroker buildFromConfig(Map ignored) { + return build(); + } + } +} diff --git a/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/js/JetStreamMessageBrokerFactory.java b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/js/JetStreamMessageBrokerFactory.java new file mode 100644 index 000000000..eaf71d95c --- /dev/null +++ b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/js/JetStreamMessageBrokerFactory.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ +package com.github.sonus21.rqueue.nats.js; + +import com.github.sonus21.rqueue.core.spi.MessageBroker; +import com.github.sonus21.rqueue.core.spi.MessageBrokerFactory; +import com.github.sonus21.rqueue.nats.RqueueNatsException; +import io.nats.client.Connection; +import io.nats.client.Nats; +import io.nats.client.Options; +import java.io.IOException; +import java.util.Map; +import java.util.Objects; + +/** + * ServiceLoader-discovered factory for the {@code "nats"} backend. Configuration keys (all + * optional except {@code rqueue.nats.url}): + * + *

    + *
  • {@code rqueue.nats.url} - NATS URL, e.g. {@code nats://localhost:4222}
    • + *
  • {@code rqueue.nats.username} / {@code rqueue.nats.password} - basic auth
    • + *
  • {@code rqueue.nats.token} - token auth
    • + *
  • {@code rqueue.nats.connectionName} - friendly client name
    • + *
+ */ +public class JetStreamMessageBrokerFactory implements MessageBrokerFactory { + + @Override + public String name() { + return "nats"; + } + + @Override + public MessageBroker create(Map config) { + Objects.requireNonNull(config, "config must not be null"); + String url = config.getOrDefault("rqueue.nats.url", Options.DEFAULT_URL); + String username = config.get("rqueue.nats.username"); + String password = config.get("rqueue.nats.password"); + String token = config.get("rqueue.nats.token"); + String connectionName = config.get("rqueue.nats.connectionName"); + + Options.Builder ob = new Options.Builder().server(url); + if (connectionName != null) { + ob.connectionName(connectionName); + } + if (token != null && !token.isEmpty()) { + ob.token(token.toCharArray()); + } else if (username != null && password != null) { + ob.userInfo(username, password); + } + Connection connection; + try { + connection = Nats.connect(ob.build()); + } catch (IOException | InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RqueueNatsException("Failed to connect to NATS at " + url, e); + } + return JetStreamMessageBroker.builder().connection(connection).build(); + } +} diff --git a/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/js/NatsStreamValidator.java b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/js/NatsStreamValidator.java new file mode 100644 index 000000000..c99a5e337 --- /dev/null +++ b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/js/NatsStreamValidator.java @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ +package com.github.sonus21.rqueue.nats.js; + +import com.github.sonus21.rqueue.config.RqueueConfig; +import com.github.sonus21.rqueue.core.EndpointRegistry; +import com.github.sonus21.rqueue.listener.QueueDetail; +import com.github.sonus21.rqueue.nats.RqueueNatsConfig; +import com.github.sonus21.rqueue.nats.RqueueNatsException; +import com.github.sonus21.rqueue.nats.internal.NatsProvisioner; +import com.github.sonus21.rqueue.utils.Constants; +import com.github.sonus21.rqueue.utils.PriorityUtils; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.springframework.beans.factory.SmartInitializingSingleton; + +/** + * Boot-time JetStream stream / DLQ existence guard. Mirrors the role + * {@code NatsKvBucketValidator} plays for KV buckets — moves stream existence checks off the + * publish / pop hot path (where they cost a {@code getStreamInfo} round-trip per message) onto + * the bootstrap path so a running broker never has to ask "does this stream exist?" again. + * + *

When this runs. Implements {@link SmartInitializingSingleton} so provisioning fires + * after every singleton bean — including {@code RqueueMessageListenerContainer} — has finished + * its {@code afterPropertiesSet}, which is when {@link EndpointRegistry} is populated, and + * before {@code SmartLifecycle.start()} spawns the message pollers. Listening on + * {@code RqueueBootstrapEvent} would race against the pollers because that event fires + * after {@code doStart()} has already submitted them, and a poll on a not-yet-created + * stream surfaces as {@code stream not found [10059]}. {@code InitializingBean} would be too + * early — the registry is still empty when this bean's own {@code afterPropertiesSet} would run. + * + *

What it walks. For every queue in {@link EndpointRegistry#getActiveQueueDetails()}: + *

    + *
  • the main stream {@code }, + *
  • one stream per declared priority sub-queue ({@code -}), + *
  • when the listener declared a Rqueue-level DLQ ({@link QueueDetail#isDlqSet()}): the + * target DLQ queue's stream ({@code }), so that + * {@code PostProcessingHandler} can publish there after retry exhaustion. + *
  • when no Rqueue DLQ is declared and {@code RqueueNatsConfig.isAutoCreateDlqStream()} is + * true: the NATS-native DLQ stream ({@code }) as a + * safety net for messages that exhaust JetStream {@code maxDeliver}. + *
+ * + *

Behaviour by flag. All work is delegated to + * {@link NatsProvisioner#ensureStream(String, java.util.List)} / + * {@link NatsProvisioner#ensureDlqStream(String, java.util.List)}, so the validator inherits the + * existing flag semantics without re-implementing them: + *

    + *
  • {@code autoCreateStreams=true} (default) — any missing stream is created using + * {@link RqueueNatsConfig.StreamDefaults}. + *
  • {@code autoCreateStreams=false} — every missing stream surfaces an + * {@link RqueueNatsException}; the validator collects all of them and raises one + * {@link IllegalStateException} listing every missing stream so operators can run a + * single batch of {@code nats stream add} commands rather than chase failures one queue + * at a time. + *
+ */ +public class NatsStreamValidator implements SmartInitializingSingleton { + + private static final Logger log = Logger.getLogger(NatsStreamValidator.class.getName()); + + private final NatsProvisioner provisioner; + private final RqueueNatsConfig config; + private final RqueueConfig rqueueConfig; + + public NatsStreamValidator(NatsProvisioner provisioner, RqueueNatsConfig config) { + this(provisioner, config, null); + } + + public NatsStreamValidator( + NatsProvisioner provisioner, RqueueNatsConfig config, RqueueConfig rqueueConfig) { + this.provisioner = provisioner; + this.config = config; + this.rqueueConfig = rqueueConfig; + } + + @Override + public void afterSingletonsInstantiated() { + List queues = EndpointRegistry.getActiveQueueDetails(); + if (queues.isEmpty()) { + log.log(Level.FINE, "NatsStreamValidator: no active queues registered; nothing to do"); + return; + } + RqueueNatsConfig.ConsumerDefaults cd = config.getConsumerDefaults(); + boolean producerOnly = rqueueConfig != null && rqueueConfig.isProducer(); + List failures = new ArrayList<>(); + int total = 0; + for (QueueDetail q : queues) { + String mainStream = config.getStreamPrefix() + q.getName(); + String mainSubject = config.getSubjectPrefix() + q.getName(); + total += tryEnsure(failures, mainStream, mainSubject, q); + if (!producerOnly) { + tryEnsureConsumer(failures, mainStream, q.resolvedConsumerName(), q, cd); + } + + if (q.getPriority() != null) { + for (String priority : q.getPriority().keySet()) { + if (Constants.DEFAULT_PRIORITY_KEY.equals(priority)) { + continue; // DEFAULT entry is the queue itself; already handled above + } + String pStream = + config.getStreamPrefix() + q.getName() + PriorityUtils.getSuffix(priority); + String pSubject = + config.getSubjectPrefix() + q.getName() + PriorityUtils.getSuffix(priority); + total += tryEnsure(failures, pStream, pSubject, q); + // Consumer is NOT created here: each priority sub-queue has its own QueueDetail + // in the registry and is processed as its own mainStream entry above, so exactly + // one consumer is created per stream. Adding a second one here would fail on + // WorkQueue streams (error 10099). + } + } + + if (q.isDlqSet()) { + // User declared a Rqueue-level DLQ: ensure the target queue's JetStream stream exists + // so that PostProcessingHandler can publish to it after retry exhaustion. The + // NATS-native "job-queue-dlq" stream is unrelated and must not be created here — + // Rqueue routes the message explicitly, not via advisory bridging. + String dlqQueueStream = config.getStreamPrefix() + q.getDeadLetterQueueName(); + String dlqQueueSubject = config.getSubjectPrefix() + q.getDeadLetterQueueName(); + total += tryEnsure(failures, dlqQueueStream, dlqQueueSubject, q); + // No consumer needed for the DLQ stream here — the DLQ queue registers its own listener. + } + } + if (!failures.isEmpty()) { + String hint = config.isAutoCreateStreams() + ? "Stream creation failed — verify NATS is running with JetStream enabled" + + " (start the server with `nats-server -js`) and that the account has" + + " `add_stream` permission." + : "With rqueue.nats.auto-create-streams=false every required stream must exist" + + " before the application starts. Run `nats stream add` for each missing" + + " stream or set rqueue.nats.auto-create-streams=true to let rqueue create" + + " them automatically."; + throw new IllegalStateException("NATS JetStream provisioning failed for " + + failures.size() + + " of " + + total + + " stream(s) at startup. " + + hint + + " Failed streams:\n" + + " - " + + String.join("\n - ", failures)); + } + log.log( + Level.INFO, + "NatsStreamValidator: ensured {0} JetStream stream(s) across {1} queue(s)", + new Object[] {total, queues.size()}); + } + + private void tryEnsureConsumer( + List failures, + String streamName, + String consumerName, + QueueDetail q, + RqueueNatsConfig.ConsumerDefaults cd) { + Duration ackWait = JetStreamMessageBroker.resolveAckWait(q, config); + long maxDeliver = JetStreamMessageBroker.resolveMaxDeliver(q, config); + try { + provisioner.ensureConsumer( + streamName, consumerName, ackWait, maxDeliver, cd.getMaxAckPending()); + } catch (RqueueNatsException e) { + failures.add("consumer " + consumerName + " on " + streamName + ": " + rootCause(e)); + } + } + + private int tryEnsure(List failures, String streamName, String subject, QueueDetail q) { + try { + provisioner.ensureStream( + streamName, List.of(subject), q.getType(), "rqueue queue: " + q.getName()); + return 1; + } catch (RqueueNatsException e) { + failures.add(streamName + " (subject " + subject + "): " + rootCause(e)); + return 1; + } + } + + private int tryEnsureDlq(List failures, String dlqStream, String dlqSubject) { + try { + provisioner.ensureDlqStream(dlqStream, List.of(dlqSubject)); + return 1; + } catch (RqueueNatsException e) { + failures.add(dlqStream + " (DLQ subject " + dlqSubject + "): " + rootCause(e)); + return 1; + } + } + + /** Returns the deepest non-null message in the cause chain for diagnostics. */ + private static String rootCause(Throwable t) { + Throwable cause = t; + while (cause.getCause() != null) { + cause = cause.getCause(); + } + String msg = cause.getMessage(); + return (msg != null && !msg.isEmpty()) ? msg : cause.getClass().getSimpleName(); + } +} diff --git a/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/kv/NatsKvBucketValidator.java b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/kv/NatsKvBucketValidator.java new file mode 100644 index 000000000..cc13e8f97 --- /dev/null +++ b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/kv/NatsKvBucketValidator.java @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ + +package com.github.sonus21.rqueue.nats.kv; + +import io.nats.client.Connection; +import io.nats.client.JetStreamApiException; +import io.nats.client.KeyValueManagement; +import io.nats.client.api.KeyValueStatus; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.springframework.beans.factory.InitializingBean; + +/** + * Startup-time guard for deployments where the application credentials cannot create JetStream + * KV buckets at runtime. + * + *

Where this runs. The Spring Boot auto-config calls {@link #validate(Connection, + * boolean)} inline in the {@code natsConnection} bean factory method, so validation + * happens during {@code Connection} bean creation — strictly before any other NATS-coupled + * bean (broker, daos, registry, lock manager) can be wired with the connection. Spring's + * {@code @Order}/{@code @Priority} only affect collection injection ordering, not bean + * creation order, so we anchor the check on the dependency root instead. + * + *

Bean form. The auto-config also declares this class as a Spring bean named + * {@code natsKvBucketValidator} so that every NATS-coupled bean can {@code @DependsOn} it. + * The bean implements {@link InitializingBean} and re-runs the same validation on its + * {@code afterPropertiesSet}; this is a cheap belt-and-suspenders check (when buckets already + * exist, all calls are {@code getStatus} no-ops). The class itself is config-source-agnostic + * — it takes the {@code autoCreate} flag as a constructor argument, so reading + * {@code rqueue.nats.autoCreateKvBuckets} from any property source is the responsibility of + * the caller (in Spring Boot, that's {@code RqueueNatsProperties}). + * + *

When {@code rqueue.nats.autoCreateKvBuckets=true} (default) the validator is a no-op: + * the existing lazy-create paths in each store / dao remain in charge. + * + *

When {@code rqueue.nats.autoCreateKvBuckets=false} the validator walks + * {@link NatsKvBuckets#ALL_BUCKETS} via {@link KeyValueManagement#getStatus(String)} and aborts + * boot with an {@link IllegalStateException} listing every missing bucket. This converts a + * subtle, late-binding "permission violation on first use" failure into a deterministic + * startup failure with operator-facing remediation (the README NATS section lists the + * {@code nats kv add} commands). + * + *

The validator does NOT pre-create buckets on its own — auto-creation, when enabled, stays + * lazy because some buckets (jobs, locks) only learn their TTL on first write. If you need + * eager creation with knowable TTLs, follow up by extending this class. + */ +public class NatsKvBucketValidator implements InitializingBean { + + private static final Logger log = Logger.getLogger(NatsKvBucketValidator.class.getName()); + + private final Connection connection; + private final boolean autoCreate; + + public NatsKvBucketValidator(Connection connection, boolean autoCreate) { + this.connection = connection; + this.autoCreate = autoCreate; + } + + @Override + public void afterPropertiesSet() { + validate(connection, autoCreate); + } + + /** + * Canonical entry point. Call this before exposing the {@link Connection} to any other + * Rqueue bean: the Spring Boot auto-config does so inside the {@code natsConnection} bean + * factory method so the connection is already validated by the time the broker, daos, + * registry, and lock manager beans inject it. + * + * @throws IllegalStateException if {@code autoCreate} is {@code false} and any required KV + * bucket is missing, or if the JetStream KV management API is unreachable. + */ + public static void validate(Connection connection, boolean autoCreate) { + if (autoCreate) { + log.fine("rqueue.nats.autoCreateKvBuckets=true; skipping startup KV bucket validation, stores" + + " will lazily create buckets as needed."); + return; + } + KeyValueManagement kvm; + try { + kvm = connection.keyValueManagement(); + } catch (IOException io) { + throw new IllegalStateException( + "Failed to obtain JetStream KeyValueManagement from NATS connection while validating" + + " KV buckets. Check that JetStream is enabled and reachable.", + io); + } + List missing = new ArrayList<>(); + for (String bucket : NatsKvBuckets.ALL_BUCKETS) { + if (!exists(kvm, bucket)) { + missing.add(bucket); + } + } + if (missing.isEmpty()) { + log.info("All required NATS KV buckets are present: " + NatsKvBuckets.ALL_BUCKETS); + return; + } + throw new IllegalStateException( + "rqueue.nats.autoCreateKvBuckets=false but the following NATS KV buckets are missing: " + + String.join(", ", missing) + + ". Pre-create them via `nats kv add ...` (see the 'NATS backend' section of the" + + " README for ready-made commands) and restart, or set" + + " rqueue.nats.autoCreateKvBuckets=true to allow Rqueue to create them at runtime."); + } + + private static boolean exists(KeyValueManagement kvm, String bucket) { + try { + KeyValueStatus status = kvm.getStatus(bucket); + return status != null; + } catch (JetStreamApiException missing) { + return false; + } catch (IOException io) { + // Surface IO problems separately — the operator can't fix a missing bucket if the + // connection itself is broken. + log.log(Level.WARNING, "KV status check for bucket " + bucket + " failed", io); + return false; + } + } +} diff --git a/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/kv/NatsKvBuckets.java b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/kv/NatsKvBuckets.java new file mode 100644 index 000000000..edab867a2 --- /dev/null +++ b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/kv/NatsKvBuckets.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ + +package com.github.sonus21.rqueue.nats.kv; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * Single source of truth for the JetStream KV bucket names the NATS backend depends on. Each + * store / dao references the constant here instead of hard-coding its own copy, and + * {@link NatsKvBucketValidator} walks {@link #ALL_BUCKETS} at + * startup to fail fast when {@code rqueue.nats.autoCreateKvBuckets=false} and any bucket is + * missing. + * + *

If you add a new KV-backed store, add its bucket name here. + */ +public final class NatsKvBuckets { + + /** Per-queue {@code QueueConfig} records (registered queues, DLQ wiring, flags). */ + public static final String QUEUE_CONFIG = "rqueue-queue-config"; + + /** {@code RqueueJob} execution history per message id. */ + public static final String JOBS = "rqueue-jobs"; + + /** Distributed locks (scheduler leadership, message-level locks). */ + public static final String LOCKS = "rqueue-locks"; + + /** Per-message metadata (delivery status, retry count, dead-letter flags). */ + public static final String MESSAGE_METADATA = "rqueue-message-metadata"; + + /** Worker process info (host, pid, version, last-seen). */ + public static final String WORKERS = "rqueue-workers"; + + /** Per-(queue, worker) heartbeats. Keys flattened as {@code __}. */ + public static final String WORKER_HEARTBEATS = "rqueue-worker-heartbeats"; + + /** Per-queue daily execution statistics (success, discard, DLQ, retry, run-time). */ + public static final String QUEUE_STATS = "rqueue-queue-stats"; + + /** All buckets the NATS backend will use, in stable order. */ + public static final List ALL_BUCKETS = Collections.unmodifiableList(Arrays.asList( + QUEUE_CONFIG, JOBS, LOCKS, MESSAGE_METADATA, WORKERS, WORKER_HEARTBEATS, QUEUE_STATS)); + + private NatsKvBuckets() {} +} diff --git a/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/lock/NatsRqueueLockManager.java b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/lock/NatsRqueueLockManager.java new file mode 100644 index 000000000..b19b77f4a --- /dev/null +++ b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/lock/NatsRqueueLockManager.java @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ + +package com.github.sonus21.rqueue.nats.lock; + +import com.github.sonus21.rqueue.common.RqueueLockManager; +import com.github.sonus21.rqueue.config.NatsBackendCondition; +import com.github.sonus21.rqueue.nats.internal.NatsProvisioner; +import com.github.sonus21.rqueue.nats.kv.NatsKvBuckets; +import io.nats.client.JetStreamApiException; +import io.nats.client.KeyValue; +import io.nats.client.api.KeyValueEntry; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.DependsOn; +import org.springframework.stereotype.Component; + +/** + * NATS-backed {@link RqueueLockManager} using a JetStream KV bucket as the lock store. + * + *

Acquire is implemented via {@link KeyValue#create} which writes only when the key doesn't + * exist (revision == 0); on existing-key conflict the JetStream server rejects the write with + * {@code wrong last sequence} and the lock is reported as not acquired. The bucket is created + * lazily with the {@code duration} of the first acquire used as the TTL — long-running keys + * past the TTL get garbage-collected by the server, so an orphaned lock from a crashed holder + * eventually self-releases. + * + *

Release verifies that the stored value matches the caller's {@code lockValue} before + * deleting, so a holder cannot release a lock another process re-acquired after expiry. + */ +@Component +@Conditional(NatsBackendCondition.class) +@DependsOn("natsKvBucketValidator") +public class NatsRqueueLockManager implements RqueueLockManager { + + private static final Logger log = Logger.getLogger(NatsRqueueLockManager.class.getName()); + private static final String BUCKET_NAME = NatsKvBuckets.LOCKS; + + private final NatsProvisioner provisioner; + + public NatsRqueueLockManager(NatsProvisioner provisioner) { + this.provisioner = provisioner; + } + + @Override + public boolean acquireLock(String lockKey, String lockValue, Duration duration) { + try { + KeyValue kv = provisioner.ensureKv(BUCKET_NAME, duration); + kv.create(sanitize(lockKey), lockValue.getBytes(StandardCharsets.UTF_8)); + return true; + } catch (JetStreamApiException existing) { + // Most common path: key already exists; another holder owns the lock. + return false; + } catch (IOException io) { + log.log(Level.WARNING, "acquireLock " + lockKey + " I/O failure", io); + return false; + } catch (RuntimeException rt) { + log.log(Level.WARNING, "acquireLock " + lockKey + " failed", rt); + return false; + } + } + + @Override + public boolean releaseLock(String lockKey, String lockValue) { + try { + KeyValue kv = provisioner.ensureKv(BUCKET_NAME, Duration.ofSeconds(60)); + String key = sanitize(lockKey); + KeyValueEntry entry = kv.get(key); + if (entry == null) { + return false; + } + String stored = new String(entry.getValue(), StandardCharsets.UTF_8); + if (!stored.equals(lockValue)) { + return false; + } + kv.delete(key, entry.getRevision()); + return true; + } catch (IOException | JetStreamApiException e) { + log.log(Level.WARNING, "releaseLock " + lockKey + " failed", e); + return false; + } + } + + /** + * KV keys allow {@code [A-Za-z0-9_=.-]} only; coerce other characters to {@code _} so the + * caller can pass arbitrary lock keys. {@code $} and {@code #} surface in queue/listener + * names from inner classes and the legacy separator respectively. + */ + private static String sanitize(String key) { + return key == null ? "_" : key.replaceAll("[^A-Za-z0-9_=.-]", "_"); + } +} diff --git a/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/metrics/NatsRqueueQueueMetricsProvider.java b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/metrics/NatsRqueueQueueMetricsProvider.java new file mode 100644 index 000000000..8e026f03b --- /dev/null +++ b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/metrics/NatsRqueueQueueMetricsProvider.java @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ +package com.github.sonus21.rqueue.nats.metrics; + +import com.github.sonus21.rqueue.core.EndpointRegistry; +import com.github.sonus21.rqueue.exception.QueueDoesNotExist; +import com.github.sonus21.rqueue.listener.QueueDetail; +import com.github.sonus21.rqueue.metrics.RqueueQueueMetricsProvider; +import com.github.sonus21.rqueue.nats.RqueueNatsConfig; +import com.github.sonus21.rqueue.nats.RqueueNatsException; +import io.nats.client.JetStreamApiException; +import io.nats.client.JetStreamManagement; +import io.nats.client.api.StreamInfo; +import java.io.IOException; + +/** + * NATS JetStream {@link RqueueQueueMetricsProvider}. Pending messages map to the message count of + * the queue's stream ({@code }). NATS does not natively support delayed + * delivery in this module, so {@link #getScheduledMessageCount(String)} always returns {@code 0}. + * + *

If the underlying stream has not yet been provisioned (no enqueue has happened), both methods + * return {@code 0} rather than failing — callers use the values as gauge readings. + */ +public class NatsRqueueQueueMetricsProvider implements RqueueQueueMetricsProvider { + + private final JetStreamManagement jsm; + private final RqueueNatsConfig config; + + public NatsRqueueQueueMetricsProvider(JetStreamManagement jsm, RqueueNatsConfig config) { + this.jsm = jsm; + this.config = config; + } + + @Override + public long getPendingMessageCount(String queueName) { + QueueDetail q; + try { + q = EndpointRegistry.get(queueName); + } catch (QueueDoesNotExist e) { + // unknown queue name -> 0 (mirrors how RedisRqueueQueueMetricsProvider handles it) + return 0L; + } + String stream = config.getStreamPrefix() + q.getName(); + try { + StreamInfo info = jsm.getStreamInfo(stream); + return info.getStreamState().getMsgCount(); + } catch (JetStreamApiException e) { + // stream not yet provisioned -> nothing pending + return 0L; + } catch (IOException e) { + throw new RqueueNatsException( + "Failed to read stream size for queue=" + queueName + " stream=" + stream, e); + } + } + + @Override + public long getScheduledMessageCount(String queueName) { + // JetStream backend does not support delayed enqueue; nothing is ever scheduled. + return 0L; + } + + @Override + public long getProcessingMessageCount(String queueName) { + // JetStream tracks in-flight messages on the consumer rather than as a separate queue. We + // don't expose this depth in v1; report 0 to keep the gauge well-defined. + return 0L; + } + + @Override + public long getDeadLetterMessageCount(String queueName) { + QueueDetail q; + try { + q = EndpointRegistry.get(queueName); + } catch (QueueDoesNotExist e) { + return 0L; + } + String dlqStream = config.getStreamPrefix() + q.getName() + config.getDlqStreamSuffix(); + try { + StreamInfo info = jsm.getStreamInfo(dlqStream); + return info.getStreamState().getMsgCount(); + } catch (JetStreamApiException e) { + // DLQ stream not provisioned -> nothing dead-lettered yet + return 0L; + } catch (IOException e) { + throw new RqueueNatsException( + "Failed to read DLQ stream size for queue=" + queueName + " stream=" + dlqStream, e); + } + } +} diff --git a/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/package-info.java b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/package-info.java new file mode 100644 index 000000000..38cfd1d64 --- /dev/null +++ b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/package-info.java @@ -0,0 +1,32 @@ +/** + * NATS / JetStream backend for Rqueue (preview, v1). + * + *

This module provides only the {@code MessageBroker} SPI implementation backed by NATS + * JetStream and a small builder API for plain-Java use. It has no dependency on Spring or Spring + * Boot. Spring wiring lives in {@code rqueue-spring} (via {@code RqueueListenerConfig} + + * {@code @EnableRqueue}); Spring Boot auto-config lives in {@code rqueue-spring-boot-starter}. + * Both gate their NATS code behind {@code @ConditionalOnClass(io.nats.client.JetStream.class)} + * and {@code @ConditionalOnProperty(name = "rqueue.backend", havingValue = "nats")}, so adding + * the {@code rqueue-nats} jar plus setting {@code rqueue.backend=nats} is sufficient to switch. + * + *

Supported in v1

+ *
    + *
  • Immediate enqueue, ack, retry-with-delay (via {@code nak}), DLQ (via {@code MaxDeliver}).
    • + *
  • Competing consumers across processes (shared durable pull consumer).
    • + *
  • Independent consumers per {@link com.github.sonus21.rqueue.annotation.RqueueListener} + * method (replaces the Redis "fan-out" pattern; the {@code @RqueueHandler} annotation is + * ignored on this backend).
    • + *
  • Message-id deduplication via {@code Nats-Msg-Id} headers.
    • + *
  • Reactive enqueue via {@code JetStream.publishAsync}.
    • + *
  • Dashboard peek via short-lived ephemeral consumers.
    • + *
+ * + *

Not supported in v1

+ *
    + *
  • Delayed / scheduled / cron messages. Calls to {@code enqueueIn}, {@code enqueueAt}, + * {@code addMessageWithDelay}, and periodic-listener registration throw {@link + * UnsupportedOperationException}.
    • + *
  • Push consumers and {@code DeliverGroup} routing. All workers use pull, durable consumers.
    • + *
+ */ +package com.github.sonus21.rqueue.nats; diff --git a/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/repository/NatsMessageBrowsingRepository.java b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/repository/NatsMessageBrowsingRepository.java new file mode 100644 index 000000000..c950ea007 --- /dev/null +++ b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/repository/NatsMessageBrowsingRepository.java @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ + +package com.github.sonus21.rqueue.nats.repository; + +import com.github.sonus21.rqueue.exception.BackendCapabilityException; +import com.github.sonus21.rqueue.models.enums.DataType; +import com.github.sonus21.rqueue.models.response.DataViewResponse; +import com.github.sonus21.rqueue.nats.RqueueNatsConfig; +import com.github.sonus21.rqueue.repository.MessageBrowsingRepository; +import io.nats.client.JetStreamApiException; +import io.nats.client.JetStreamManagement; +import io.nats.client.api.StreamInfo; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * NATS-backend impl of {@link MessageBrowsingRepository}. Maps Redis-style queue-name keys to + * JetStream streams and returns actual message counts from JetStream {@link StreamInfo}. The + * size queries provide real values for: + * + *
    + *
  • Main queue pending: looks up stream by extracting queue name and prefixing with stream + * prefix + *
  • Dead-letter counts: looks up DLQ stream with DLQ suffix + *
  • Processing/Scheduled queues: returns 0 (NATS doesn't expose these separately; consumers + * tracks in-flight implicitly, scheduled delivery unsupported) + *
+ * + *

{@link #viewData} throws {@link BackendCapabilityException} (mapped to HTTP 501 by the web + * advice) since JetStream KV doesn't expose positional reads on arbitrary keys; the + * data-explorer panel is Redis-only in v1. + */ +public class NatsMessageBrowsingRepository implements MessageBrowsingRepository { + + private static final Logger log = Logger.getLogger(NatsMessageBrowsingRepository.class.getName()); + + private final JetStreamManagement jsm; + private final RqueueNatsConfig config; + + public NatsMessageBrowsingRepository(JetStreamManagement jsm, RqueueNatsConfig config) { + this.jsm = jsm; + this.config = config; + } + + @Override + public long getDataSize(String name, DataType type) { + if (name == null || name.isEmpty()) { + return 0L; + } + try { + // Try to extract queue name from Redis-style key patterns and look up the NATS stream. + String queueName = extractQueueName(name); + if (queueName == null) { + return 0L; // Key pattern not recognized; return 0. + } + + // Check if this is a DLQ by matching against known DLQ patterns. + if (isDlqName(name, queueName)) { + String dlqStream = config.getStreamPrefix() + queueName + config.getDlqStreamSuffix(); + return getStreamMessageCount(dlqStream); + } + + // For main queue, processing, and scheduled queue patterns, return sizes. + // Processing/scheduled counts return 0 in v1 (not exposed by NATS/JetStream natively). + if (isProcessingQueue(name) || isScheduledQueue(name)) { + return 0L; + } + + // Assume it's a main queue pending count. + String stream = config.getStreamPrefix() + queueName; + return getStreamMessageCount(stream); + } catch (IOException | JetStreamApiException e) { + log.log(Level.WARNING, "Failed to get data size for name=" + name, e); + return 0L; // Fail gracefully; return 0 if stream lookup fails. + } + } + + @Override + public List getDataSizes(List names, List types) { + if (names == null || names.isEmpty()) { + return new ArrayList<>(); + } + List out = new ArrayList<>(names.size()); + for (String name : names) { + out.add(getDataSize(name, null)); + } + return out; + } + + @Override + public DataViewResponse viewData( + String name, DataType type, String key, int pageNumber, int itemPerPage) { + throw new BackendCapabilityException( + "nats", + "viewData", + "JetStream does not expose positional reads on arbitrary keys; the dashboard's" + + " data-explorer panel is Redis-only in v1."); + } + + /** + * Extract the base queue name from a Redis-style key. Recognizes patterns like: + *

    + *
  • {@code __rq::queue::queueName} → {@code queueName} + *
  • {@code __rq::p-queue::queueName} → {@code queueName} + *
  • {@code __rq::d-queue::queueName} → {@code queueName} + *
  • {@code queueName-dlq} → {@code queueName} (if it's a DLQ) + *
  • Other patterns → {@code null} + *
+ */ + private String extractQueueName(String name) { + // Pattern: __rq::queue::queueName or __rq::p-queue::queueName or __rq::d-queue::queueName + if (name.startsWith("__rq::")) { + int lastSeparator = name.lastIndexOf("::"); + if (lastSeparator >= 0) { + return name.substring(lastSeparator + 2); + } + } + // Assume it's a direct queue name (e.g., for DLQ patterns). + return name; + } + + /** + * Check if the name matches a processing-queue pattern ({@code __rq::p-queue::...}). + */ + private boolean isProcessingQueue(String name) { + return name != null && name.contains("::p-queue::"); + } + + /** + * Check if the name matches a scheduled-queue pattern ({@code __rq::d-queue::...}). + */ + private boolean isScheduledQueue(String name) { + return name != null && name.contains("::d-queue::"); + } + + /** + * Check if the extracted queue name matches a DLQ name pattern. For now, we assume any name not + * matching the {@code __rq::} prefix patterns is a DLQ candidate. + */ + private boolean isDlqName(String originalName, String extractedQueueName) { + return !originalName.startsWith("__rq::"); + } + + /** + * Safely query a JetStream stream for its message count. Returns 0 if the stream doesn't + * exist or on errors. + */ + private long getStreamMessageCount(String streamName) throws IOException, JetStreamApiException { + try { + StreamInfo info = jsm.getStreamInfo(streamName); + return info.getStreamState().getMsgCount(); + } catch (JetStreamApiException e) { + // 10059 = stream not found; return 0 rather than failing. + if (e.getApiErrorCode() == 10059) { + return 0L; + } + throw e; + } + } +} diff --git a/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/service/NatsRqueueMessageMetadataService.java b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/service/NatsRqueueMessageMetadataService.java new file mode 100644 index 000000000..252402621 --- /dev/null +++ b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/service/NatsRqueueMessageMetadataService.java @@ -0,0 +1,240 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ + +package com.github.sonus21.rqueue.nats.service; + +import com.github.sonus21.rqueue.config.NatsBackendCondition; +import com.github.sonus21.rqueue.core.RqueueMessage; +import com.github.sonus21.rqueue.core.support.RqueueMessageUtils; +import com.github.sonus21.rqueue.models.db.MessageMetadata; +import com.github.sonus21.rqueue.models.enums.MessageStatus; +import com.github.sonus21.rqueue.nats.internal.NatsProvisioner; +import com.github.sonus21.rqueue.nats.kv.NatsKvBuckets; +import com.github.sonus21.rqueue.service.RqueueMessageMetadataService; +import io.nats.client.JetStreamApiException; +import io.nats.client.KeyValue; +import io.nats.client.api.KeyValueEntry; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.DependsOn; +import org.springframework.data.redis.core.ZSetOperations.TypedTuple; +import org.springframework.stereotype.Service; +import reactor.core.publisher.Mono; + +/** + * NATS-backed {@link RqueueMessageMetadataService} using a JetStream KV bucket as the metadata + * store. Entries are keyed by metadata id (which the Redis impl computes via + * {@link RqueueMessageUtils#getMessageMetaId}) and serialized as JSON. + * + *

Per-queue read methods ({@link #readMessageMetadataForQueue}) walk the bucket; rqueue + * normally tracks recent metadata only so the volume is acceptable for v1. The Redis impl keeps + * a per-queue ZSET as an explicit reverse index — that's a follow-up here. + * + *

{@link #saveReactive} wraps the synchronous {@code save} in a {@code Mono}; rqueue's + * reactive enqueue path is the only caller and short-circuits storeMessageMetadata when the + * broker has {@code !usesPrimaryHandlerDispatch}, so this method is rarely hit in practice. + */ +@Service +@Conditional(NatsBackendCondition.class) +@DependsOn("natsKvBucketValidator") +public class NatsRqueueMessageMetadataService implements RqueueMessageMetadataService { + + private static final Logger log = + Logger.getLogger(NatsRqueueMessageMetadataService.class.getName()); + private static final String BUCKET_NAME = NatsKvBuckets.MESSAGE_METADATA; + + private final NatsProvisioner provisioner; + private final com.github.sonus21.rqueue.serdes.RqueueSerDes serdes; + + public NatsRqueueMessageMetadataService( + NatsProvisioner provisioner, com.github.sonus21.rqueue.serdes.RqueueSerDes serdes) { + this.provisioner = provisioner; + this.serdes = serdes; + } + + private KeyValue kv() throws IOException, JetStreamApiException { + return provisioner.ensureKv(BUCKET_NAME, null); + } + + @Override + public MessageMetadata get(String id) { + return loadByKey(sanitize(id)); + } + + @Override + public void delete(String id) { + try { + kv().delete(sanitize(id)); + } catch (IOException | JetStreamApiException e) { + log.log(Level.WARNING, "delete metadata " + id + " failed", e); + } + } + + @Override + public void deleteAll(Collection ids) { + for (String id : ids) { + delete(id); + } + } + + @Override + public List findAll(Collection ids) { + List out = new ArrayList<>(ids.size()); + for (String id : ids) { + MessageMetadata m = get(id); + if (m != null) { + out.add(m); + } + } + return out; + } + + @Override + public void save(MessageMetadata messageMetadata, Duration ttl, boolean checkUnique) { + try { + kv().put(sanitize(messageMetadata.getId()), serialize(messageMetadata)); + } catch (IOException | JetStreamApiException e) { + log.log(Level.WARNING, "save metadata " + messageMetadata.getId() + " failed", e); + } + } + + @Override + public MessageMetadata getByMessageId(String queueName, String messageId) { + return get(RqueueMessageUtils.getMessageMetaId(queueName, messageId)); + } + + @Override + public boolean deleteMessage(String queueName, String messageId, Duration ttl) { + String metaId = RqueueMessageUtils.getMessageMetaId(queueName, messageId); + MessageMetadata m = get(metaId); + if (m == null) { + return false; + } + m.setDeleted(true); + m.setDeletedOn(System.currentTimeMillis()); + save(m, ttl, false); + return true; + } + + @Override + public MessageMetadata getOrCreateMessageMetadata(RqueueMessage rqueueMessage) { + String metaId = + RqueueMessageUtils.getMessageMetaId(rqueueMessage.getQueueName(), rqueueMessage.getId()); + MessageMetadata existing = get(metaId); + return existing != null ? existing : new MessageMetadata(rqueueMessage, MessageStatus.ENQUEUED); + } + + @Override + public Mono saveReactive(MessageMetadata m, Duration ttl, boolean checkUnique) { + return Mono.fromCallable(() -> { + save(m, ttl, checkUnique); + return Boolean.TRUE; + }); + } + + @Override + public List> readMessageMetadataForQueue( + String queueName, long start, long end) { + // The Redis impl uses a ZSET sorted by createdAt for pagination. We don't have a sorted + // index here, so this returns all metadata for the queue and the caller paginates. + try { + List keys = new ArrayList<>(kv().keys()); + List> out = new ArrayList<>(); + String prefix = sanitize(queueName); + for (String k : keys) { + if (!k.startsWith(prefix)) { + continue; + } + MessageMetadata m = loadByKey(k); + if (m != null) { + out.add(TypedTuple.of(m, (double) m.getUpdatedOn())); + } + } + return out; + } catch (IOException | JetStreamApiException | InterruptedException e) { + log.log(Level.WARNING, "readMessageMetadataForQueue " + queueName + " failed", e); + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + return Collections.emptyList(); + } + } + + @Override + public void saveMessageMetadataForQueue( + String queueName, MessageMetadata messageMetadata, Long ttlInMillisecond) { + save( + messageMetadata, + ttlInMillisecond == null ? null : Duration.ofMillis(ttlInMillisecond), + false); + } + + @Override + public void deleteQueueMessages(String queueName, long before) { + try { + List keys = new ArrayList<>(kv().keys()); + String prefix = sanitize(queueName); + for (String k : keys) { + if (!k.startsWith(prefix)) { + continue; + } + MessageMetadata m = loadByKey(k); + if (m != null && m.getUpdatedOn() < before) { + kv().delete(k); + } + } + } catch (IOException | JetStreamApiException | InterruptedException e) { + log.log(Level.WARNING, "deleteQueueMessages " + queueName + " failed", e); + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + } + } + + // ---- helpers ---------------------------------------------------------- + + private MessageMetadata loadByKey(String key) { + try { + KeyValueEntry entry = kv().get(key); + if (entry == null || entry.getValue() == null) { + return null; + } + return deserialize(entry.getValue()); + } catch (IOException | JetStreamApiException e) { + log.log(Level.WARNING, "loadByKey " + key + " failed", e); + return null; + } + } + + private byte[] serialize(MessageMetadata m) throws IOException { + return serdes.serialize(m); + } + + private MessageMetadata deserialize(byte[] bytes) { + try { + return serdes.deserialize(bytes, MessageMetadata.class); + } catch (Exception e) { + log.log(Level.WARNING, "deserialize MessageMetadata failed", e); + return null; + } + } + + /** KV keys allow {@code [A-Za-z0-9_=.-]} only. */ + private static String sanitize(String key) { + return key == null ? "_" : key.replaceAll("[^A-Za-z0-9_=.-]", "_"); + } +} diff --git a/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/service/NatsRqueueUtilityService.java b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/service/NatsRqueueUtilityService.java new file mode 100644 index 000000000..b7fdc3f3a --- /dev/null +++ b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/service/NatsRqueueUtilityService.java @@ -0,0 +1,191 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ + +package com.github.sonus21.rqueue.nats.service; + +import com.github.sonus21.rqueue.config.NatsBackendCondition; +import com.github.sonus21.rqueue.config.RqueueWebConfig; +import com.github.sonus21.rqueue.models.Pair; +import com.github.sonus21.rqueue.models.enums.AggregationType; +import com.github.sonus21.rqueue.models.request.MessageMoveRequest; +import com.github.sonus21.rqueue.models.request.PauseUnpauseQueueRequest; +import com.github.sonus21.rqueue.models.response.BaseResponse; +import com.github.sonus21.rqueue.models.response.BooleanResponse; +import com.github.sonus21.rqueue.models.response.DataSelectorResponse; +import com.github.sonus21.rqueue.models.response.MessageMoveResponse; +import com.github.sonus21.rqueue.models.response.StringResponse; +import com.github.sonus21.rqueue.service.RqueueUtilityService; +import com.github.sonus21.rqueue.utils.Constants; +import java.util.LinkedList; +import java.util.List; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Conditional; +import org.springframework.stereotype.Service; +import reactor.core.publisher.Mono; + +/** + * NATS-backend stub for {@link RqueueUtilityService}. Admin/dashboard utility methods are + * Redis-only in v1; this stub returns "not supported" responses uniformly so the rest of the + * bean graph stays consistent. Replace with a NATS-native implementation in a follow-up. + */ +@Service +@Conditional(NatsBackendCondition.class) +public class NatsRqueueUtilityService implements RqueueUtilityService { + + @Autowired + private RqueueWebConfig rqueueWebConfig; + + private static T notSupported(T response, String op) { + response.setCode(1); + response.setMessage(op + " is not supported with rqueue.backend=nats in v1"); + return response; + } + + @Override + public BooleanResponse deleteMessage(String queueName, String id) { + return notSupported(new BooleanResponse(), "deleteMessage"); + } + + @Override + public BooleanResponse enqueueMessage(String queueName, String id, String position) { + return notSupported(new BooleanResponse(), "enqueueMessage"); + } + + @Override + public MessageMoveResponse moveMessage(MessageMoveRequest messageMoveRequest) { + return notSupported(new MessageMoveResponse(), "moveMessage"); + } + + @Override + public BooleanResponse makeEmpty(String queueName, String dataName) { + return notSupported(new BooleanResponse(), "makeEmpty"); + } + + @Override + public Pair getLatestVersion() { + return new Pair<>("", ""); + } + + @Override + public StringResponse getDataType(String name) { + return notSupported(new StringResponse(), "getDataType"); + } + + @Override + public Mono makeEmptyReactive(String queueName, String datasetName) { + return Mono.just(notSupported(new BooleanResponse(), "makeEmptyReactive")); + } + + @Override + public Mono deleteReactiveMessage(String queueName, String messageId) { + return Mono.just(notSupported(new BooleanResponse(), "deleteReactiveMessage")); + } + + @Override + public Mono enqueueReactiveMessage( + String queueName, String messageId, String position) { + return Mono.just(notSupported(new BooleanResponse(), "enqueueReactiveMessage")); + } + + @Override + public Mono getReactiveDataType(String name) { + return Mono.just(notSupported(new StringResponse(), "getReactiveDataType")); + } + + @Override + public Mono moveReactiveMessage(MessageMoveRequest request) { + return Mono.just(notSupported(new MessageMoveResponse(), "moveReactiveMessage")); + } + + @Override + public Mono reactivePauseUnpauseQueue(PauseUnpauseQueueRequest request) { + return Mono.just(notSupported(new BaseResponse(), "reactivePauseUnpauseQueue")); + } + + @Override + public BaseResponse pauseUnpauseQueue(PauseUnpauseQueueRequest request) { + return notSupported(new BaseResponse(), "pauseUnpauseQueue"); + } + + @Override + public Mono reactiveAggregateDataCounter(AggregationType type) { + return Mono.just(aggregateDataCounter(type)); + } + + @Override + public DataSelectorResponse aggregateDataCounter(AggregationType type) { + String title; + List> data; + if (type == AggregationType.DAILY) { + data = getDailyDateCounter(); + title = "Select Number of Days"; + } else if (type == AggregationType.WEEKLY) { + data = getWeeklyDateCounter(); + title = "Select Number of Weeks"; + } else { + data = getMonthlyDateCounter(); + title = "Select Number of Months"; + } + return new DataSelectorResponse(title, data); + } + + private List> getDailyDateCounter() { + List> dateSelector = new LinkedList<>(); + int[] dates = new int[] {1, 2, 3, 4, 6, 7}; + int step = 15; + int stepAfter = 15; + int i = 1; + dateSelector.add(new Pair<>("0", "Select")); + while (i <= rqueueWebConfig.getHistoryDay()) { + if (i >= stepAfter) { + if (i <= rqueueWebConfig.getHistoryDay()) { + dateSelector.add(new Pair<>(String.valueOf(i), String.format("Last %d days", i))); + } + i += step; + } else { + for (int date : dates) { + if (date == i) { + String suffix = i == 1 ? "day" : "days"; + dateSelector.add( + new Pair<>(String.valueOf(date), String.format("Last %d %s", date, suffix))); + break; + } + } + i += 1; + } + } + return dateSelector; + } + + private List> getWeeklyDateCounter() { + List> dateSelector = new LinkedList<>(); + dateSelector.add(new Pair<>("0", "Select")); + int nWeek = + (int) Math.ceil(rqueueWebConfig.getHistoryDay() / (double) Constants.DAYS_IN_A_WEEK); + for (int week = 1; week <= nWeek; week++) { + String suffix = week == 1 ? "week" : "weeks"; + dateSelector.add(new Pair<>(String.valueOf(week), String.format("Last %d %s", week, suffix))); + } + return dateSelector; + } + + private List> getMonthlyDateCounter() { + List> dateSelector = new LinkedList<>(); + dateSelector.add(new Pair<>("0", "Select")); + int nMonths = + (int) Math.ceil(rqueueWebConfig.getHistoryDay() / (double) Constants.DAYS_IN_A_MONTH); + for (int month = 1; month <= nMonths; month++) { + String suffix = month == 1 ? "month" : "months"; + dateSelector.add( + new Pair<>(String.valueOf(month), String.format("Last %d %s", month, suffix))); + } + return dateSelector; + } +} diff --git a/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/worker/NatsWorkerRegistryStore.java b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/worker/NatsWorkerRegistryStore.java new file mode 100644 index 000000000..0db7ee62d --- /dev/null +++ b/rqueue-nats/src/main/java/com/github/sonus21/rqueue/nats/worker/NatsWorkerRegistryStore.java @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ + +package com.github.sonus21.rqueue.nats.worker; + +import com.github.sonus21.rqueue.config.NatsBackendCondition; +import com.github.sonus21.rqueue.models.registry.RqueueWorkerInfo; +import com.github.sonus21.rqueue.nats.internal.NatsProvisioner; +import com.github.sonus21.rqueue.nats.kv.NatsKvBuckets; +import com.github.sonus21.rqueue.worker.WorkerRegistryStore; +import io.nats.client.JetStreamApiException; +import io.nats.client.KeyValue; +import io.nats.client.api.KeyValueEntry; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.DependsOn; +import org.springframework.stereotype.Repository; + +/** + * NATS JetStream KV-backed {@link WorkerRegistryStore}. Uses two buckets so each can carry its + * own bucket-level {@code maxAge}: {@code rqueue-workers} (worker info, TTL = workerTtl) and + * {@code rqueue-worker-heartbeats} (per-(queue, workerId) heartbeats, TTL = queueTtl). + * + *

Per-queue heartbeats are stored as flattened keys of the form + * {@code "__"}. Listing heartbeats for a queue iterates + * all bucket keys with the matching {@code __} prefix. + * + *

Bucket {@code ttl} (NATS' name for entry max-age) is a one-shot configuration set at bucket + * creation; {@link #refreshQueueTtl(String, Duration)} is therefore a no-op — every write into + * the heartbeat bucket re-establishes the entry's age from zero, which is sufficient given the + * registry rewrites heartbeats on the configured interval. + * + *

The first call lazily creates each bucket. {@code ttl} is fixed at bucket creation, so + * existing buckets are reused even if the configured TTL has since changed. + */ +@Repository +@Conditional(NatsBackendCondition.class) +@DependsOn("natsKvBucketValidator") +public class NatsWorkerRegistryStore implements WorkerRegistryStore { + + private static final Logger log = Logger.getLogger(NatsWorkerRegistryStore.class.getName()); + private static final String WORKER_BUCKET = NatsKvBuckets.WORKERS; + private static final String HEARTBEAT_BUCKET = NatsKvBuckets.WORKER_HEARTBEATS; + /** Separator used to flatten a {@code (queueKey, workerId)} pair into a single KV key. */ + private static final String SEP = "__"; + + private final NatsProvisioner provisioner; + private final com.github.sonus21.rqueue.serdes.RqueueSerDes serdes; + /** Captured on first putWorkerInfo call so worker bucket gets the right maxAge. */ + private volatile Duration workerBucketTtl; + /** Captured on first putQueueHeartbeat / refreshQueueTtl so heartbeat bucket gets the right maxAge. */ + private volatile Duration heartbeatBucketTtl; + + public NatsWorkerRegistryStore( + NatsProvisioner provisioner, com.github.sonus21.rqueue.serdes.RqueueSerDes serdes) { + this.provisioner = provisioner; + this.serdes = serdes; + } + + @Override + public void putWorkerInfo(String workerKey, RqueueWorkerInfo info, Duration ttl) { + if (workerBucketTtl == null) { + workerBucketTtl = ttl; + } + try { + KeyValue kv = provisioner.ensureKv(WORKER_BUCKET, workerBucketTtl); + kv.put(sanitize(workerKey), serialize(info)); + } catch (IOException | JetStreamApiException e) { + log.log(Level.WARNING, "putWorkerInfo " + workerKey + " failed", e); + } + } + + @Override + public void deleteWorkerInfo(String workerKey) { + try { + KeyValue kv = provisioner.ensureKv(WORKER_BUCKET, workerBucketTtl); + kv.delete(sanitize(workerKey)); + } catch (IOException | JetStreamApiException e) { + log.log(Level.WARNING, "deleteWorkerInfo " + workerKey + " failed", e); + } + } + + @Override + public Map getWorkerInfos(Collection workerKeys) { + if (workerKeys == null || workerKeys.isEmpty()) { + return Collections.emptyMap(); + } + Map out = new LinkedHashMap<>(); + try { + KeyValue kv = provisioner.ensureKv(WORKER_BUCKET, workerBucketTtl); + for (String key : workerKeys) { + KeyValueEntry entry = kv.get(sanitize(key)); + if (entry == null || entry.getValue() == null) { + continue; + } + RqueueWorkerInfo info = deserialize(entry.getValue()); + if (info != null && info.getWorkerId() != null) { + out.put(info.getWorkerId(), info); + } + } + } catch (IOException | JetStreamApiException e) { + log.log(Level.WARNING, "getWorkerInfos failed", e); + } + return out; + } + + @Override + public void putQueueHeartbeat(String queueKey, String workerId, String metadataJson) { + if (heartbeatBucketTtl == null) { + // Best-effort default — overwritten by refreshQueueTtl when the registry computes the real + // value. + heartbeatBucketTtl = Duration.ofHours(1); + } + try { + KeyValue kv = provisioner.ensureKv(HEARTBEAT_BUCKET, heartbeatBucketTtl); + kv.put(compositeKey(queueKey, workerId), metadataJson.getBytes()); + } catch (IOException | JetStreamApiException e) { + log.log(Level.WARNING, "putQueueHeartbeat queue=" + queueKey + " failed", e); + } + } + + @Override + public Map getQueueHeartbeats(String queueKey) { + Map out = new LinkedHashMap<>(); + try { + KeyValue kv = provisioner.ensureKv(HEARTBEAT_BUCKET, heartbeatBucketTtl); + String prefix = sanitize(queueKey) + SEP; + List keys = new ArrayList<>(kv.keys()); + for (String k : keys) { + if (!k.startsWith(prefix)) { + continue; + } + KeyValueEntry entry = kv.get(k); + if (entry == null || entry.getValue() == null) { + continue; + } + String workerId = k.substring(prefix.length()); + out.put(workerId, new String(entry.getValue())); + } + } catch (IOException | JetStreamApiException | InterruptedException e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + log.log(Level.WARNING, "getQueueHeartbeats queue=" + queueKey + " failed", e); + } + return out; + } + + @Override + public void deleteQueueHeartbeats(String queueKey, String... workerIds) { + if (workerIds == null || workerIds.length == 0) { + return; + } + try { + KeyValue kv = provisioner.ensureKv(HEARTBEAT_BUCKET, heartbeatBucketTtl); + for (String workerId : workerIds) { + kv.delete(compositeKey(queueKey, workerId)); + } + } catch (IOException | JetStreamApiException e) { + log.log(Level.WARNING, "deleteQueueHeartbeats queue=" + queueKey + " failed", e); + } + } + + @Override + public void refreshQueueTtl(String queueKey, Duration ttl) { + // NATS KV applies maxAge at the bucket level and resets per-entry age on each write. + // The registry already rewrites heartbeats on its configured interval, so each fresh put + // implicitly resets expiry. We only capture the first observed ttl so the bucket created + // on first use carries the correct maxAge. + if (heartbeatBucketTtl == null) { + heartbeatBucketTtl = ttl; + } + } + + // ---- helpers ---------------------------------------------------------- + + private static String compositeKey(String queueKey, String workerId) { + return sanitize(queueKey) + SEP + sanitize(workerId); + } + + /** KV keys allow {@code [A-Za-z0-9_=.-]} only. */ + private static String sanitize(String key) { + return key == null ? "_" : key.replaceAll("[^A-Za-z0-9_=.-]", "_"); + } + + private byte[] serialize(RqueueWorkerInfo info) throws IOException { + return serdes.serialize(info); + } + + private RqueueWorkerInfo deserialize(byte[] bytes) { + try { + return serdes.deserialize(bytes, RqueueWorkerInfo.class); + } catch (Exception e) { + log.log(Level.WARNING, "deserialize RqueueWorkerInfo failed", e); + return null; + } + } +} diff --git a/rqueue-core/src/main/resources/META-INF/spring-configuration-metadata.json b/rqueue-nats/src/main/resources/META-INF/additional-spring-configuration-metadata.json similarity index 84% rename from rqueue-core/src/main/resources/META-INF/spring-configuration-metadata.json rename to rqueue-nats/src/main/resources/META-INF/additional-spring-configuration-metadata.json index 7f7c484b2..26041b37d 100644 --- a/rqueue-core/src/main/resources/META-INF/spring-configuration-metadata.json +++ b/rqueue-nats/src/main/resources/META-INF/additional-spring-configuration-metadata.json @@ -1,5 +1,19 @@ { - "hints" : [], + "hints" : [ + { + "name" : "rqueue.backend", + "values" : [ + { + "value" : "redis", + "description" : "Redis backend (default). Requires a RedisConnectionFactory bean." + }, + { + "value" : "nats", + "description" : "NATS JetStream backend. Requires the rqueue-nats module and a running NATS server with JetStream enabled." + } + ] + } + ], "groups" : [ { "sourceType" : "com.github.sonus21.rqueue.spring.boot.RqueueMetricsProperties", @@ -232,6 +246,62 @@ "type" : "com.github.sonus21.rqueue.models.enums.RqueueMode", "defaultValue" : "BOTH" }, + { + "sourceType" : "com.github.sonus21.rqueue.config.RqueueConfig", + "name" : "rqueue.backend", + "description" : "Message broker backend. Use 'redis' (default) for the Redis backend or 'nats' to switch to NATS JetStream. The 'nats' value requires the rqueue-nats module on the classpath.", + "type" : "java.lang.String", + "defaultValue" : "redis" + }, + { + "sourceType" : "com.github.sonus21.rqueue.config.RqueueConfig", + "name" : "rqueue.worker.registry.enabled", + "description" : "Enable worker and queue-poller tracking for the dashboard", + "type" : "java.lang.Boolean", + "defaultValue" : true + }, + { + "sourceType" : "com.github.sonus21.rqueue.config.RqueueConfig", + "name" : "rqueue.worker.registry.worker.ttl", + "description" : "TTL in seconds for worker metadata entries", + "type" : "java.lang.Long", + "defaultValue" : 300 + }, + { + "sourceType" : "com.github.sonus21.rqueue.config.RqueueConfig", + "name" : "rqueue.worker.registry.worker.heartbeat.interval", + "description" : "Interval in seconds for refreshing worker metadata", + "type" : "java.lang.Long", + "defaultValue" : 60 + }, + { + "sourceType" : "com.github.sonus21.rqueue.config.RqueueConfig", + "name" : "rqueue.worker.registry.queue.ttl", + "description" : "TTL in seconds for queue-poller metadata entries", + "type" : "java.lang.Long", + "defaultValue" : 3600 + }, + { + "sourceType" : "com.github.sonus21.rqueue.config.RqueueConfig", + "name" : "rqueue.worker.registry.queue.heartbeat.interval", + "description" : "Interval in seconds for queue-poller heartbeat refresh", + "type" : "java.lang.Long", + "defaultValue" : 15 + }, + { + "sourceType" : "com.github.sonus21.rqueue.config.RqueueConfig", + "name" : "rqueue.worker.registry.key.prefix", + "description" : "Key prefix for worker registry entries", + "type" : "java.lang.String", + "defaultValue" : "worker::" + }, + { + "sourceType" : "com.github.sonus21.rqueue.config.RqueueConfig", + "name" : "rqueue.worker.registry.queue.key.prefix", + "description" : "Key prefix for queue-poller registry entries", + "type" : "java.lang.String", + "defaultValue" : "q-pollers::" + }, { "sourceType" : "com.github.sonus21.rqueue.config.RqueueConfig", "name" : "rqueue.message.converter.provider.class", diff --git a/rqueue-nats/src/main/resources/META-INF/services/com.github.sonus21.rqueue.core.spi.MessageBrokerFactory b/rqueue-nats/src/main/resources/META-INF/services/com.github.sonus21.rqueue.core.spi.MessageBrokerFactory new file mode 100644 index 000000000..2dc523269 --- /dev/null +++ b/rqueue-nats/src/main/resources/META-INF/services/com.github.sonus21.rqueue.core.spi.MessageBrokerFactory @@ -0,0 +1 @@ +com.github.sonus21.rqueue.nats.js.JetStreamMessageBrokerFactory diff --git a/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/AbstractJetStreamIT.java b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/AbstractJetStreamIT.java new file mode 100644 index 000000000..2831edb94 --- /dev/null +++ b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/AbstractJetStreamIT.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ +package com.github.sonus21.rqueue.nats; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.github.sonus21.rqueue.enums.QueueType; +import com.github.sonus21.rqueue.listener.QueueDetail; +import io.nats.client.Connection; +import io.nats.client.Nats; +import io.nats.client.Options; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; + +/** + * Base for JetStream integration tests. Mirrors the Redis test pattern: when {@code NATS_RUNNING} + * is set the test connects to a locally running nats-server (CI path); otherwise a Testcontainers- + * managed instance is started in {@link BeforeAll} (local Docker path). JUnit 5 / Testcontainers + * skip the test gracefully if Docker isn't available and {@code NATS_RUNNING} isn't set. + */ +@Testcontainers(disabledWithoutDocker = true) +@NatsIntegrationTest +public abstract class AbstractJetStreamIT { + + static final boolean USE_EXTERNAL_NATS = System.getenv("NATS_RUNNING") != null; + static final String EXTERNAL_NATS_URL = + System.getenv().getOrDefault("NATS_URL", "nats://127.0.0.1:4222"); + + /** + * Container is only constructed in the local-Docker path. The Testcontainers extension + * ignores static {@code GenericContainer} fields that aren't annotated {@code @Container}; + * we manage the container's lifecycle from {@link #setup()} / {@link #teardown()} so the + * external-NATS path can leave it null without tripping the extension. + */ + protected static GenericContainer NATS; + + protected static Connection connection; + + @BeforeAll + static void setup() throws Exception { + String url; + if (USE_EXTERNAL_NATS) { + url = EXTERNAL_NATS_URL; + } else { + NATS = new GenericContainer<>(DockerImageName.parse("nats:2.10-alpine")) + .withCommand("-js", "-DV") + .withExposedPorts(4222) + .waitingFor(Wait.forLogMessage(".*Server is ready.*\\n", 1)); + NATS.start(); + url = "nats://" + NATS.getHost() + ":" + NATS.getMappedPort(4222); + } + connection = Nats.connect(new Options.Builder().server(url).build()); + } + + @AfterAll + static void teardown() throws Exception { + if (connection != null) { + connection.close(); + } + if (NATS != null && NATS.isRunning()) { + NATS.stop(); + } + } + + protected QueueDetail mockQueue(String name) { + return mockQueue(name, QueueType.QUEUE); + } + + protected QueueDetail mockQueue(String name, QueueType type) { + QueueDetail q = mock(QueueDetail.class); + when(q.getName()).thenReturn(name); + when(q.getType()).thenReturn(type); + return q; + } +} diff --git a/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerCompetingConsumersIT.java b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerCompetingConsumersIT.java new file mode 100644 index 000000000..b253d2307 --- /dev/null +++ b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerCompetingConsumersIT.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ +package com.github.sonus21.rqueue.nats; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.github.sonus21.rqueue.core.RqueueMessage; +import com.github.sonus21.rqueue.listener.QueueDetail; +import com.github.sonus21.rqueue.nats.js.JetStreamMessageBroker; +import java.time.Duration; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import org.junit.jupiter.api.Test; + +@NatsIntegrationTest +class JetStreamMessageBrokerCompetingConsumersIT extends AbstractJetStreamIT { + + @Test + void twoWorkersSharingDurable_eachMessageDeliveredOnce() throws Exception { + QueueDetail q = mockQueue("ccq-" + System.nanoTime()); + int total = 20; + try (JetStreamMessageBroker broker = + JetStreamMessageBroker.builder().connection(connection).build()) { + for (int i = 0; i < total; i++) { + broker.enqueue(q, RqueueMessage.builder().id("m-" + i).message("p" + i).build()); + } + Set seen = ConcurrentHashMap.newKeySet(); + CountDownLatch done = new CountDownLatch(total); + var pool = Executors.newFixedThreadPool(2); + for (int t = 0; t < 2; t++) { + pool.submit(() -> { + for (int round = 0; round < 50 && done.getCount() > 0; round++) { + List popped = broker.pop(q, "shared", 5, Duration.ofMillis(500)); + for (RqueueMessage m : popped) { + if (seen.add(m.getId())) { + done.countDown(); + } + broker.ack(q, m); + } + } + }); + } + done.await(20, java.util.concurrent.TimeUnit.SECONDS); + pool.shutdownNow(); + assertEquals(total, seen.size(), "every message should be seen exactly once across workers"); + } + } +} diff --git a/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerDedupIT.java b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerDedupIT.java new file mode 100644 index 000000000..02bebdb6b --- /dev/null +++ b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerDedupIT.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ +package com.github.sonus21.rqueue.nats; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.github.sonus21.rqueue.core.RqueueMessage; +import com.github.sonus21.rqueue.listener.QueueDetail; +import com.github.sonus21.rqueue.nats.js.JetStreamMessageBroker; +import org.junit.jupiter.api.Test; + +@NatsIntegrationTest +class JetStreamMessageBrokerDedupIT extends AbstractJetStreamIT { + + @Test + void duplicateMsgIdInsideWindow_isDeduped() throws Exception { + QueueDetail q = mockQueue("ddq-" + System.nanoTime()); + try (JetStreamMessageBroker broker = + JetStreamMessageBroker.builder().connection(connection).build()) { + RqueueMessage m1 = RqueueMessage.builder().id("dup-1").message("a").build(); + RqueueMessage m2 = RqueueMessage.builder().id("dup-1").message("b").build(); + broker.enqueue(q, m1); + broker.enqueue(q, m2); + assertEquals(1L, broker.size(q)); + } + } +} diff --git a/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerDelayThrowsTest.java b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerDelayThrowsTest.java new file mode 100644 index 000000000..9a66d63ce --- /dev/null +++ b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerDelayThrowsTest.java @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ +package com.github.sonus21.rqueue.nats; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.github.sonus21.rqueue.core.RqueueMessage; +import com.github.sonus21.rqueue.core.spi.Capabilities; +import com.github.sonus21.rqueue.listener.QueueDetail; +import com.github.sonus21.rqueue.nats.js.JetStreamMessageBroker; +import com.github.sonus21.rqueue.serdes.RqJacksonSerDes; +import com.github.sonus21.rqueue.serdes.SerializationUtils; +import io.nats.client.Connection; +import io.nats.client.JetStream; +import io.nats.client.JetStreamManagement; +import org.junit.jupiter.api.Test; + +/** Unit tests that exercise pure-Java code paths (no docker container needed). */ +@NatsUnitTest +class JetStreamMessageBrokerDelayThrowsTest { + + private JetStreamMessageBroker newBroker() { + Connection conn = mock(Connection.class); + JetStream js = mock(JetStream.class); + JetStreamManagement jsm = mock(JetStreamManagement.class); + return new JetStreamMessageBroker( + conn, + js, + jsm, + RqueueNatsConfig.defaults(), + new RqJacksonSerDes(SerializationUtils.getObjectMapper()), + null); + } + + @Test + void enqueueWithDelay_throwsUOE() { + JetStreamMessageBroker broker = newBroker(); + QueueDetail q = mock(QueueDetail.class); + when(q.getName()).thenReturn("orders"); + RqueueMessage m = RqueueMessage.builder().id("id-1").message("hi").build(); + UnsupportedOperationException ex = + assertThrows(UnsupportedOperationException.class, () -> broker.enqueueWithDelay(q, m, 100)); + // message must mention NATS so users grep'ing for UOE find a useful pointer + org.junit.jupiter.api.Assertions.assertTrue(ex.getMessage().toLowerCase().contains("nats")); + } + + @Test + void capabilities_delayAndCronFalse_primaryDispatchFalse() { + Capabilities caps = newBroker().capabilities(); + assertEquals(false, caps.supportsDelayedEnqueue()); + assertEquals(false, caps.supportsScheduledIntrospection()); + assertEquals(false, caps.supportsCronJobs()); + // NATS uses its own JetStream subscription dispatch, not the Redis primary handler path. + assertEquals(false, caps.usesPrimaryHandlerDispatch()); + } + + @Test + void moveExpired_isNoOpReturningZero() { + JetStreamMessageBroker broker = newBroker(); + QueueDetail q = mock(QueueDetail.class); + when(q.getName()).thenReturn("orders"); + assertEquals(0L, broker.moveExpired(q, System.currentTimeMillis(), 100)); + } + + @Test + void ack_withoutInFlight_returnsFalse() { + JetStreamMessageBroker broker = newBroker(); + QueueDetail q = mock(QueueDetail.class); + when(q.getName()).thenReturn("orders"); + assertFalse(broker.ack(q, RqueueMessage.builder().id("never-popped").build())); + assertFalse(broker.nack(q, RqueueMessage.builder().id("never-popped").build(), 100)); + } +} diff --git a/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerEnqueueAckIT.java b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerEnqueueAckIT.java new file mode 100644 index 000000000..accef105d --- /dev/null +++ b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerEnqueueAckIT.java @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ +package com.github.sonus21.rqueue.nats; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.github.sonus21.rqueue.core.RqueueMessage; +import com.github.sonus21.rqueue.listener.QueueDetail; +import com.github.sonus21.rqueue.nats.js.JetStreamMessageBroker; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.Test; + +@NatsIntegrationTest +class JetStreamMessageBrokerEnqueueAckIT extends AbstractJetStreamIT { + + @Test + void enqueuePopAck_drainsStream() throws Exception { + QueueDetail q = mockQueue("eaq-" + System.nanoTime()); + RqueueNatsConfig cfg = RqueueNatsConfig.defaults(); + cfg.getStreamDefaults().setRetention(io.nats.client.api.RetentionPolicy.WorkQueue); + try (JetStreamMessageBroker broker = + JetStreamMessageBroker.builder().connection(connection).config(cfg).build()) { + List sent = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + RqueueMessage m = + RqueueMessage.builder().id("m-" + i).message("payload-" + i).build(); + broker.enqueue(q, m); + sent.add(m); + } + assertEquals(10L, broker.size(q)); + + int received = 0; + for (int round = 0; round < 5 && received < 10; round++) { + List popped = broker.pop(q, "worker", 4, Duration.ofSeconds(2)); + for (RqueueMessage m : popped) { + assertTrue(broker.ack(q, m), "ack should succeed for " + m.getId()); + received++; + } + } + assertEquals(10, received); + // WorkQueue retention removes acked msgs from the stream, but JetStream processes ACKs + // asynchronously: nm.ack() is fire-and-forget, so size() can briefly observe non-zero + // until the server applies the deletion. Poll for drain instead of asserting strictly. + long deadlineNanos = System.nanoTime() + Duration.ofSeconds(5).toNanos(); + long size = broker.size(q); + while (size != 0L && System.nanoTime() < deadlineNanos) { + Thread.sleep(50L); + size = broker.size(q); + } + assertEquals(0L, size); + } + } +} diff --git a/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerFactoryTest.java b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerFactoryTest.java new file mode 100644 index 000000000..d2a3f652f --- /dev/null +++ b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerFactoryTest.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ +package com.github.sonus21.rqueue.nats; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.github.sonus21.rqueue.core.spi.MessageBrokerFactory; +import com.github.sonus21.rqueue.core.spi.MessageBrokerLoader; +import com.github.sonus21.rqueue.nats.js.JetStreamMessageBrokerFactory; +import java.util.HashMap; +import java.util.Map; +import java.util.ServiceLoader; +import org.junit.jupiter.api.Test; + +/** ServiceLoader and configuration-parsing tests for {@link JetStreamMessageBrokerFactory}. */ +@NatsUnitTest +class JetStreamMessageBrokerFactoryTest { + + /** Use a port that isn't bound; jnats should fail fast trying to connect. */ + private static Map unreachableConfig() { + Map cfg = new HashMap<>(); + cfg.put("rqueue.nats.url", "nats://127.0.0.1:1"); + return cfg; + } + + @Test + void name_isLowercaseNats() { + assertEquals("nats", new JetStreamMessageBrokerFactory().name()); + } + + @Test + void serviceLoader_findsJetStreamFactory() { + JetStreamMessageBrokerFactory found = null; + for (MessageBrokerFactory f : ServiceLoader.load(MessageBrokerFactory.class)) { + if ("nats".equals(f.name()) && f instanceof JetStreamMessageBrokerFactory) { + found = (JetStreamMessageBrokerFactory) f; + break; + } + } + assertNotNull(found, "ServiceLoader did not discover JetStreamMessageBrokerFactory"); + } + + @Test + void create_withUnreachableUrl_throwsRqueueNatsException() { + JetStreamMessageBrokerFactory factory = new JetStreamMessageBrokerFactory(); + RqueueNatsException ex = + assertThrows(RqueueNatsException.class, () -> factory.create(unreachableConfig())); + // message should reference NATS and the URL so failures are debuggable + String msg = ex.getMessage(); + assertNotNull(msg); + assertTrue(msg.toLowerCase().contains("nats"), "message should mention NATS: " + msg); + assertTrue(msg.contains("127.0.0.1:1"), "message should include URL: " + msg); + } + + @Test + void create_withNullConfig_throwsNpe() { + JetStreamMessageBrokerFactory factory = new JetStreamMessageBrokerFactory(); + assertThrows(NullPointerException.class, () -> factory.create(null)); + } + + @Test + void messageBrokerLoader_load_natsBackend_routesToJetStreamFactory() { + // Same exception expected as direct factory.create() since URL is unreachable. + RqueueNatsException ex = assertThrows( + RqueueNatsException.class, () -> MessageBrokerLoader.load("nats", unreachableConfig())); + assertTrue(ex.getMessage().toLowerCase().contains("nats")); + } + + @Test + void messageBrokerLoader_load_unknownBackend_throwsIAE() { + IllegalArgumentException ex = assertThrows( + IllegalArgumentException.class, () -> MessageBrokerLoader.load("nope", new HashMap<>())); + assertTrue( + ex.getMessage().contains("No MessageBrokerFactory found"), + "expected 'No MessageBrokerFactory found' in message but was: " + ex.getMessage()); + } +} diff --git a/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerIndependentConsumersIT.java b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerIndependentConsumersIT.java new file mode 100644 index 000000000..cf25cf19c --- /dev/null +++ b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerIndependentConsumersIT.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ +package com.github.sonus21.rqueue.nats; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.github.sonus21.rqueue.core.RqueueMessage; +import com.github.sonus21.rqueue.enums.QueueType; +import com.github.sonus21.rqueue.listener.QueueDetail; +import com.github.sonus21.rqueue.nats.js.JetStreamMessageBroker; +import java.time.Duration; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.junit.jupiter.api.Test; + +@NatsIntegrationTest +class JetStreamMessageBrokerIndependentConsumersIT extends AbstractJetStreamIT { + + @Test + void twoDurables_eachReceiveAllMessages() throws Exception { + QueueDetail q = mockQueue("icq-" + System.nanoTime(), QueueType.STREAM); + int total = 5; + try (JetStreamMessageBroker broker = + JetStreamMessageBroker.builder().connection(connection).build()) { + for (int i = 0; i < total; i++) { + broker.enqueue(q, RqueueMessage.builder().id("m-" + i).message("p" + i).build()); + } + + Set aSeen = new HashSet<>(); + Set bSeen = new HashSet<>(); + drainInto(broker, q, "consumer-a", aSeen); + drainInto(broker, q, "consumer-b", bSeen); + + assertEquals(total, aSeen.size()); + assertEquals(total, bSeen.size()); + } + } + + private void drainInto( + JetStreamMessageBroker broker, QueueDetail q, String consumer, Set sink) + throws Exception { + long deadline = System.currentTimeMillis() + 5000; + while (System.currentTimeMillis() < deadline && sink.size() < 5) { + List popped = broker.pop(q, consumer, 5, Duration.ofMillis(500)); + for (RqueueMessage m : popped) { + sink.add(m.getId()); + broker.ack(q, m); + } + } + } +} diff --git a/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerPeekIT.java b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerPeekIT.java new file mode 100644 index 000000000..9ca8dbde9 --- /dev/null +++ b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerPeekIT.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ +package com.github.sonus21.rqueue.nats; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.github.sonus21.rqueue.core.RqueueMessage; +import com.github.sonus21.rqueue.enums.QueueType; +import com.github.sonus21.rqueue.listener.QueueDetail; +import com.github.sonus21.rqueue.nats.js.JetStreamMessageBroker; +import io.nats.client.api.ConsumerInfo; +import java.util.List; +import org.junit.jupiter.api.Test; + +@NatsIntegrationTest +class JetStreamMessageBrokerPeekIT extends AbstractJetStreamIT { + + @Test + void peek_doesNotPerturbDurableConsumerAckPending() throws Exception { + // Peek creates an AckPolicy.None ephemeral consumer which WorkQueue streams reject (error + // 10084). Peek is inherently a fan-out / read-only operation and requires a Limits stream. + QueueDetail q = mockQueue("pkq-" + System.nanoTime(), QueueType.STREAM); + RqueueNatsConfig cfg = RqueueNatsConfig.defaults(); + try (JetStreamMessageBroker broker = + JetStreamMessageBroker.builder().connection(connection).config(cfg).build()) { + for (int i = 1; i <= 5; i++) { + broker.enqueue(q, RqueueMessage.builder().id("m-" + i).message("p" + i).build()); + } + // create durable consumer first so we can compare ack-pending before/after peek + broker.pop(q, "worker", 0, java.time.Duration.ofMillis(50)); // ensures consumer exists + String stream = cfg.getStreamPrefix() + q.getName(); + ConsumerInfo before = connection.jetStreamManagement().getConsumerInfo(stream, "worker"); + + List peeked = broker.peek(q, 2, 3); + assertEquals(3, peeked.size(), "expected 3 messages starting at offset 2"); + + ConsumerInfo after = connection.jetStreamManagement().getConsumerInfo(stream, "worker"); + assertEquals( + before.getNumAckPending(), + after.getNumAckPending(), + "peek must not affect durable consumer's ack-pending count"); + } + } +} diff --git a/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerProducerOnlyIT.java b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerProducerOnlyIT.java new file mode 100644 index 000000000..433a61f04 --- /dev/null +++ b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerProducerOnlyIT.java @@ -0,0 +1,187 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ +package com.github.sonus21.rqueue.nats; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.github.sonus21.rqueue.core.RqueueMessage; +import com.github.sonus21.rqueue.listener.QueueDetail; +import com.github.sonus21.rqueue.nats.js.JetStreamMessageBroker; +import com.github.sonus21.rqueue.serdes.SerializationUtils; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.junit.jupiter.api.Test; +import reactor.core.publisher.Flux; +import reactor.test.StepVerifier; +import tools.jackson.databind.ObjectMapper; + +/** + * End-to-end producer-only smoke test: the broker enqueues typed domain events but never pops or + * acks them, mirroring the producer-only application mode where the process only publishes work. + * + *

Covers plain enqueue, priority enqueue, and reactive enqueue — verifying that all variants + * land in JetStream and are reflected by {@link JetStreamMessageBroker#size}. + */ +@NatsIntegrationTest +class JetStreamMessageBrokerProducerOnlyIT extends AbstractJetStreamIT { + + private static final ObjectMapper MAPPER = SerializationUtils.objectMapper; + + // ---- minimal domain events used as message payloads -------------------- + + @Data + @NoArgsConstructor + @AllArgsConstructor + static class EmailEvent { + private String id; + private String to; + private String subject; + } + + @Data + @NoArgsConstructor + @AllArgsConstructor + static class JobEvent { + private String id; + private String type; + } + + @Data + @NoArgsConstructor + @AllArgsConstructor + static class NotificationEvent { + private String id; + private String message; + } + + // ---- helpers ----------------------------------------------------------- + + private static String serialize(Object event) throws Exception { + return MAPPER.writeValueAsString(event); + } + + private static RqueueMessage rqueueMessage(String id, Object event) throws Exception { + return RqueueMessage.builder().id(id).message(serialize(event)).build(); + } + + // ---- tests ------------------------------------------------------------- + + @Test + void enqueueEmailEvents_accumulateInStream() throws Exception { + QueueDetail emailQueue = mockQueue("email-queue-" + System.nanoTime()); + try (JetStreamMessageBroker broker = + JetStreamMessageBroker.builder().connection(connection).build()) { + int count = 5; + for (int i = 0; i < count; i++) { + EmailEvent event = new EmailEvent( + UUID.randomUUID().toString(), "user" + i + "@example.com", "Subject " + i); + broker.enqueue(emailQueue, rqueueMessage("email-" + i, event)); + } + assertEquals( + count, broker.size(emailQueue), "all email events should be visible in the stream"); + } + } + + @Test + void enqueueJobEvents_accumulateInStream() throws Exception { + QueueDetail jobQueue = mockQueue("job-queue-" + System.nanoTime()); + try (JetStreamMessageBroker broker = + JetStreamMessageBroker.builder().connection(connection).build()) { + String[] types = {"FULL_TIME", "PART_TIME", "CONTRACT"}; + for (int i = 0; i < types.length; i++) { + JobEvent event = new JobEvent(UUID.randomUUID().toString(), types[i]); + broker.enqueue(jobQueue, rqueueMessage("job-" + i, event)); + } + assertEquals( + types.length, broker.size(jobQueue), "all job events should be visible in the stream"); + } + } + + @Test + void enqueueWithPriority_notificationEvents_accumulateInPriorityStreams() throws Exception { + QueueDetail notifQueue = mockQueue("notif-queue-" + System.nanoTime()); + try (JetStreamMessageBroker broker = + JetStreamMessageBroker.builder().connection(connection).build()) { + String[] priorities = {"high", "low"}; + int perPriority = 4; + for (String priority : priorities) { + for (int i = 0; i < perPriority; i++) { + NotificationEvent event = + new NotificationEvent(UUID.randomUUID().toString(), priority + "-notification-" + i); + broker.enqueue(notifQueue, priority, rqueueMessage(priority + "-notif-" + i, event)); + } + } + for (String priority : priorities) { + QueueDetail pq = mockQueue(notifQueue.getName() + "_" + priority); + assertEquals( + perPriority, + broker.size(pq), + "priority=" + priority + " stream should hold " + perPriority + " notification events"); + } + } + } + + @Test + void enqueueReactive_emailEvents_accumulateInStream() throws Exception { + QueueDetail emailQueue = mockQueue("email-reactive-" + System.nanoTime()); + try (JetStreamMessageBroker broker = + JetStreamMessageBroker.builder().connection(connection).build()) { + int count = 6; + List messages = new ArrayList<>(); + for (int i = 0; i < count; i++) { + EmailEvent event = new EmailEvent( + UUID.randomUUID().toString(), "user" + i + "@example.com", "RE: item " + i); + messages.add(rqueueMessage("re-email-" + i, event)); + } + Flux publishes = + Flux.fromIterable(messages).flatMap(m -> broker.enqueueReactive(emailQueue, m)); + StepVerifier.create(publishes).verifyComplete(); + assertEquals( + count, + broker.size(emailQueue), + "all reactively enqueued email events should be in the stream"); + } + } + + @Test + void mixedEvents_allVariantsLandInCorrectStreams() throws Exception { + String base = "mixed-events-" + System.nanoTime(); + QueueDetail mainQueue = mockQueue(base); + QueueDetail highQueue = mockQueue(base + "_high"); + + try (JetStreamMessageBroker broker = + JetStreamMessageBroker.builder().connection(connection).build()) { + + // 3 email events on the main queue + for (int i = 0; i < 3; i++) { + EmailEvent email = + new EmailEvent(UUID.randomUUID().toString(), "to" + i + "@example.com", "Hello " + i); + broker.enqueue(mainQueue, rqueueMessage("email-" + i, email)); + } + // 2 job events on the "high" priority sub-queue + for (int i = 0; i < 2; i++) { + JobEvent job = new JobEvent(UUID.randomUUID().toString(), "CONTRACT"); + broker.enqueue(mainQueue, "high", rqueueMessage("job-high-" + i, job)); + } + // 1 notification reactively on the main queue + NotificationEvent notif = + new NotificationEvent(UUID.randomUUID().toString(), "reactive notif"); + StepVerifier.create(broker.enqueueReactive(mainQueue, rqueueMessage("notif-0", notif))) + .verifyComplete(); + + assertEquals(4L, broker.size(mainQueue), "main stream: 3 email + 1 reactive notification"); + assertEquals(2L, broker.size(highQueue), "high-priority stream: 2 job events"); + } + } +} diff --git a/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerPubSubIT.java b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerPubSubIT.java new file mode 100644 index 000000000..d50932e00 --- /dev/null +++ b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerPubSubIT.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ +package com.github.sonus21.rqueue.nats; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.github.sonus21.rqueue.nats.js.JetStreamMessageBroker; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Test; + +@NatsIntegrationTest +class JetStreamMessageBrokerPubSubIT extends AbstractJetStreamIT { + + @Test + void publishSubscribe_handlerReceivesPayload() throws Exception { + try (JetStreamMessageBroker broker = + JetStreamMessageBroker.builder().connection(connection).build()) { + ArrayBlockingQueue received = new ArrayBlockingQueue<>(4); + String channel = "test.channel." + System.nanoTime(); + try (AutoCloseable sub = broker.subscribe(channel, received::offer)) { + // small wait so the dispatcher subscription is registered server-side + Thread.sleep(100); + broker.publish(channel, "hello"); + String msg = received.poll(2, TimeUnit.SECONDS); + assertTrue(msg != null, "handler should receive a message"); + assertEquals("hello", msg); + } + } + } +} diff --git a/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerQueueNameValidationTest.java b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerQueueNameValidationTest.java new file mode 100644 index 000000000..ac04791e0 --- /dev/null +++ b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerQueueNameValidationTest.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ +package com.github.sonus21.rqueue.nats; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; + +import com.github.sonus21.rqueue.nats.js.JetStreamMessageBroker; +import com.github.sonus21.rqueue.serdes.RqJacksonSerDes; +import com.github.sonus21.rqueue.serdes.SerializationUtils; +import io.nats.client.Connection; +import io.nats.client.JetStream; +import io.nats.client.JetStreamManagement; +import org.junit.jupiter.api.Test; + +/** + * Pins the dot/wildcard/whitespace rejection on queue names for the NATS backend. NATS subjects + * use {@code .} as a hierarchy separator and stream / consumer names disallow it outright, so + * silently accepting an illegal name leads to an opaque driver-side rejection at first publish. + */ +@NatsUnitTest +class JetStreamMessageBrokerQueueNameValidationTest { + + private JetStreamMessageBroker newBroker() { + return new JetStreamMessageBroker( + mock(Connection.class), + mock(JetStream.class), + mock(JetStreamManagement.class), + RqueueNatsConfig.defaults(), + new RqJacksonSerDes(SerializationUtils.getObjectMapper()), + null); + } + + @Test + void rejects_queueName_with_dot() { + JetStreamMessageBroker broker = newBroker(); + IllegalArgumentException ex = + assertThrows(IllegalArgumentException.class, () -> broker.validateQueueName("orders.us")); + assertTrue(ex.getMessage().contains("orders.us")); + assertTrue(ex.getMessage().contains("'.'")); + } + + @Test + void rejects_queueName_with_star_wildcard() { + JetStreamMessageBroker broker = newBroker(); + assertThrows(IllegalArgumentException.class, () -> broker.validateQueueName("foo*bar")); + } + + @Test + void rejects_queueName_with_gt_wildcard() { + JetStreamMessageBroker broker = newBroker(); + assertThrows(IllegalArgumentException.class, () -> broker.validateQueueName("foo>bar")); + } + + @Test + void rejects_queueName_with_whitespace() { + JetStreamMessageBroker broker = newBroker(); + assertThrows(IllegalArgumentException.class, () -> broker.validateQueueName("foo bar")); + assertThrows(IllegalArgumentException.class, () -> broker.validateQueueName("foo\tbar")); + } + + @Test + void accepts_legal_queue_names() { + JetStreamMessageBroker broker = newBroker(); + assertDoesNotThrow(() -> broker.validateQueueName("orders")); + assertDoesNotThrow(() -> broker.validateQueueName("orders-us")); + assertDoesNotThrow(() -> broker.validateQueueName("orders_us")); + assertDoesNotThrow(() -> broker.validateQueueName("orders123")); + assertDoesNotThrow(() -> broker.validateQueueName("a")); + } + + @Test + void accepts_null_or_empty_so_that_callers_higher_up_the_stack_handle_those_errors() { + JetStreamMessageBroker broker = newBroker(); + assertDoesNotThrow(() -> broker.validateQueueName(null)); + assertDoesNotThrow(() -> broker.validateQueueName("")); + } +} diff --git a/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerReactiveEnqueueIT.java b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerReactiveEnqueueIT.java new file mode 100644 index 000000000..bcfff487c --- /dev/null +++ b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerReactiveEnqueueIT.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ +package com.github.sonus21.rqueue.nats; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.github.sonus21.rqueue.core.RqueueMessage; +import com.github.sonus21.rqueue.listener.QueueDetail; +import com.github.sonus21.rqueue.nats.js.JetStreamMessageBroker; +import org.junit.jupiter.api.Test; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.test.StepVerifier; + +@NatsIntegrationTest +class JetStreamMessageBrokerReactiveEnqueueIT extends AbstractJetStreamIT { + + @Test + void enqueueReactive_publishesAllMessages() { + QueueDetail q = mockQueue("re-" + System.nanoTime()); + try (JetStreamMessageBroker broker = + JetStreamMessageBroker.builder().connection(connection).build()) { + + Flux publishes = Flux.range(0, 5).flatMap(i -> { + RqueueMessage m = + RqueueMessage.builder().id("rm-" + i).message("payload-" + i).build(); + return broker.enqueueReactive(q, m); + }); + + StepVerifier.create(publishes).verifyComplete(); + + assertEquals(5L, broker.size(q)); + } + } + + @Test + void enqueueWithDelayReactive_returnsUnsupportedOperationException() { + QueueDetail q = mockQueue("rd-" + System.nanoTime()); + try (JetStreamMessageBroker broker = + JetStreamMessageBroker.builder().connection(connection).build()) { + RqueueMessage m = RqueueMessage.builder().id("rm-delay").message("p").build(); + + Mono mono = broker.enqueueWithDelayReactive(q, m, 1_000L); + + StepVerifier.create(mono).verifyError(UnsupportedOperationException.class); + } + } +} diff --git a/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerRetryDlqIT.java b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerRetryDlqIT.java new file mode 100644 index 000000000..6a37ea678 --- /dev/null +++ b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerRetryDlqIT.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ +package com.github.sonus21.rqueue.nats; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.github.sonus21.rqueue.core.RqueueMessage; +import com.github.sonus21.rqueue.listener.QueueDetail; +import com.github.sonus21.rqueue.nats.js.JetStreamMessageBroker; +import java.time.Duration; +import java.util.List; +import org.junit.jupiter.api.Test; + +@NatsIntegrationTest +class JetStreamMessageBrokerRetryDlqIT extends AbstractJetStreamIT { + + @Test + void exhaustedMessage_landsOnDlqStream() throws Exception { + String name = "rdq-" + System.nanoTime(); + QueueDetail q = mockQueue(name); + RqueueNatsConfig cfg = RqueueNatsConfig.defaults(); + cfg.getConsumerDefaults().setMaxDeliver(2); + cfg.getConsumerDefaults().setAckWait(Duration.ofMillis(500)); + + try (JetStreamMessageBroker broker = + JetStreamMessageBroker.builder().connection(connection).config(cfg).build()) { + // Provision DLQ + advisory bridge + broker.provisionDlq(q); + try (AutoCloseable bridge = broker.installDeadLetterBridge(q, "worker")) { + broker.enqueue(q, RqueueMessage.builder().id("retry-1").message("hi").build()); + + // Pop and nak twice (maxDeliver=2 means after 2 deliveries it's exhausted) + for (int i = 0; i < 3; i++) { + List popped = broker.pop(q, "worker", 1, Duration.ofSeconds(2)); + for (RqueueMessage m : popped) { + broker.nack(q, m, 0L); + } + Thread.sleep(600); + } + + // give the advisory bridge a moment to react + long deadline = System.currentTimeMillis() + 5000; + long dlqSize = 0; + QueueDetail dlqProbe = mockQueue(name); // size of original stream + // We don't expose dlq stream directly; verify via JSM stream lookup via a fresh probe + while (System.currentTimeMillis() < deadline) { + long s = connection + .jetStreamManagement() + .getStreamInfo(cfg.getStreamPrefix() + name + cfg.getDlqStreamSuffix()) + .getStreamState() + .getMsgCount(); + if (s > 0) { + dlqSize = s; + break; + } + Thread.sleep(200); + } + assertTrue(dlqSize >= 1, "Expected at least 1 message on DLQ stream, got " + dlqSize); + } + } + } +} diff --git a/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerStreamDescriptionTest.java b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerStreamDescriptionTest.java new file mode 100644 index 000000000..aa458ea2c --- /dev/null +++ b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerStreamDescriptionTest.java @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ +package com.github.sonus21.rqueue.nats; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.github.sonus21.rqueue.core.RqueueMessage; +import com.github.sonus21.rqueue.enums.QueueType; +import com.github.sonus21.rqueue.listener.QueueDetail; +import com.github.sonus21.rqueue.nats.internal.NatsProvisioner; +import com.github.sonus21.rqueue.nats.js.JetStreamMessageBroker; +import com.github.sonus21.rqueue.serdes.RqJacksonSerDes; +import com.github.sonus21.rqueue.serdes.SerializationUtils; +import io.nats.client.Connection; +import io.nats.client.JetStream; +import io.nats.client.JetStreamManagement; +import io.nats.client.api.PublishAck; +import io.nats.client.impl.Headers; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +/** + * Pins the stream {@code description} that JetStreamMessageBroker forwards to {@link + * NatsProvisioner#ensureStream(String, java.util.List, QueueType, String)} so {@code nats stream + * info} shows operators which rqueue queue created the stream. + */ +@NatsUnitTest +class JetStreamMessageBrokerStreamDescriptionTest { + + private static QueueDetail queueNamed(String name) { + QueueDetail q = mock(QueueDetail.class); + when(q.getName()).thenReturn(name); + when(q.getType()).thenReturn(QueueType.QUEUE); + return q; + } + + private static class Fixture { + final JetStream js; + final NatsProvisioner provisioner; + final JetStreamMessageBroker broker; + + Fixture() { + Connection conn = mock(Connection.class); + this.js = mock(JetStream.class); + JetStreamManagement jsm = mock(JetStreamManagement.class); + this.provisioner = mock(NatsProvisioner.class); + this.broker = new JetStreamMessageBroker( + conn, + js, + jsm, + RqueueNatsConfig.defaults(), + new RqJacksonSerDes(SerializationUtils.getObjectMapper()), + provisioner); + } + } + + @Test + void onQueueRegistered_passesQueueNameDescription() { + Fixture f = new Fixture(); + f.broker.onQueueRegistered(queueNamed("orders")); + verify(f.provisioner) + .ensureStream( + eq("rqueue-js-orders"), anyList(), eq(QueueType.QUEUE), eq("rqueue queue: orders")); + } + + @Test + void enqueue_passesQueueNameDescription() throws Exception { + Fixture f = new Fixture(); + when(f.js.publish(any(String.class), any(Headers.class), any(byte[].class))) + .thenReturn(mock(PublishAck.class)); + f.broker.enqueue( + queueNamed("orders"), RqueueMessage.builder().id("m1").message("hi").build()); + verify(f.provisioner) + .ensureStream( + eq("rqueue-js-orders"), anyList(), any(QueueType.class), eq("rqueue queue: orders")); + } + + @Test + void enqueueWithPriority_includesPriorityInDescription() throws Exception { + Fixture f = new Fixture(); + when(f.js.publish(any(String.class), any(Headers.class), any(byte[].class))) + .thenReturn(mock(PublishAck.class)); + f.broker.enqueue( + queueNamed("orders"), + "high", + RqueueMessage.builder().id("m1").message("hi").build()); + ArgumentCaptor desc = ArgumentCaptor.forClass(String.class); + verify(f.provisioner, atLeastOnce()) + .ensureStream(any(String.class), anyList(), any(QueueType.class), desc.capture()); + String d = desc.getValue(); + assertTrue(d.contains("orders"), "description should include queue name, was: " + d); + assertTrue(d.contains("high"), "description should include priority, was: " + d); + } +} diff --git a/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerUnitTest.java b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerUnitTest.java new file mode 100644 index 000000000..d7fb334d5 --- /dev/null +++ b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamMessageBrokerUnitTest.java @@ -0,0 +1,217 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ +package com.github.sonus21.rqueue.nats; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.github.sonus21.rqueue.core.RqueueMessage; +import com.github.sonus21.rqueue.listener.QueueDetail; +import com.github.sonus21.rqueue.nats.internal.NatsProvisioner; +import com.github.sonus21.rqueue.nats.js.JetStreamMessageBroker; +import com.github.sonus21.rqueue.serdes.RqJacksonSerDes; +import com.github.sonus21.rqueue.serdes.SerializationUtils; +import io.nats.client.Connection; +import io.nats.client.Dispatcher; +import io.nats.client.JetStream; +import io.nats.client.JetStreamApiException; +import io.nats.client.JetStreamManagement; +import io.nats.client.MessageHandler; +import io.nats.client.api.PublishAck; +import io.nats.client.impl.Headers; +import java.io.IOException; +import java.util.concurrent.CompletableFuture; +import org.junit.jupiter.api.Test; +import reactor.test.StepVerifier; + +/** + * Non-container unit tests for {@link JetStreamMessageBroker} that mock the underlying NATS + * primitives. These tests target subject naming, pub/sub plumbing, and exception wrapping — + * end-to-end JetStream behavior is covered by the Docker-gated ITs. + */ +@NatsUnitTest +class JetStreamMessageBrokerUnitTest { + + private static QueueDetail queueNamed(String name) { + QueueDetail q = mock(QueueDetail.class); + when(q.getName()).thenReturn(name); + return q; + } + + /** Build a broker with all NATS primitives mocked and stream provisioning short-circuited. */ + private static Fixture newFixture(RqueueNatsConfig config) { + Connection conn = mock(Connection.class); + JetStream js = mock(JetStream.class); + JetStreamManagement jsm = mock(JetStreamManagement.class); + // Mock the provisioner so ensureStream() is a no-op — these tests verify subject + // naming and exception wrapping, not stream creation. + NatsProvisioner provisioner = mock(NatsProvisioner.class); + JetStreamMessageBroker broker = new JetStreamMessageBroker( + conn, + js, + jsm, + config, + new RqJacksonSerDes(SerializationUtils.getObjectMapper()), + provisioner); + return new Fixture(conn, js, jsm, broker); + } + + @Test + void enqueue_publishesToPrefixedSubject() throws Exception { + Fixture f = newFixture(RqueueNatsConfig.defaults()); + when(f.js.publish(any(String.class), any(Headers.class), any(byte[].class))) + .thenReturn(mock(PublishAck.class)); + f.broker.enqueue( + queueNamed("orders"), RqueueMessage.builder().id("m1").message("hi").build()); + verify(f.js, times(1)).publish(eq("rqueue.js.orders"), any(Headers.class), any(byte[].class)); + } + + @Test + void enqueueWithPriority_appendsPrioritySuffixToSubject() throws Exception { + Fixture f = newFixture(RqueueNatsConfig.defaults()); + when(f.js.publish(any(String.class), any(Headers.class), any(byte[].class))) + .thenReturn(mock(PublishAck.class)); + f.broker.enqueue( + queueNamed("orders"), + "high", + RqueueMessage.builder().id("m1").message("hi").build()); + verify(f.js, times(1)) + .publish(eq("rqueue.js.orders_high"), any(Headers.class), any(byte[].class)); + } + + @Test + void enqueueWithEmptyPriority_fallsBackToUnsuffixedSubject() throws Exception { + Fixture f = newFixture(RqueueNatsConfig.defaults()); + when(f.js.publish(any(String.class), any(Headers.class), any(byte[].class))) + .thenReturn(mock(PublishAck.class)); + f.broker.enqueue( + queueNamed("orders"), "", RqueueMessage.builder().id("m1").message("hi").build()); + verify(f.js, times(1)).publish(eq("rqueue.js.orders"), any(Headers.class), any(byte[].class)); + } + + @Test + void enqueue_honorsCustomSubjectPrefix() throws Exception { + RqueueNatsConfig cfg = RqueueNatsConfig.defaults().setSubjectPrefix("custom."); + Fixture f = newFixture(cfg); + when(f.js.publish(any(String.class), any(Headers.class), any(byte[].class))) + .thenReturn(mock(PublishAck.class)); + f.broker.enqueue( + queueNamed("orders"), RqueueMessage.builder().id("m1").message("hi").build()); + verify(f.js, times(1)).publish(eq("custom.orders"), any(Headers.class), any(byte[].class)); + } + + @Test + void enqueue_wrapsIoExceptionInRqueueNatsException() throws Exception { + Fixture f = newFixture(RqueueNatsConfig.defaults()); + when(f.js.publish(any(String.class), any(Headers.class), any(byte[].class))) + .thenThrow(new IOException("boom")); + RqueueNatsException ex = assertThrows( + RqueueNatsException.class, + () -> f.broker.enqueue( + queueNamed("orders"), RqueueMessage.builder().id("m1").message("hi").build())); + assertNotNull(ex.getCause()); + } + + @Test + void enqueue_wrapsJetStreamApiExceptionInRqueueNatsException() throws Exception { + Fixture f = newFixture(RqueueNatsConfig.defaults()); + when(f.js.publish(any(String.class), any(Headers.class), any(byte[].class))) + .thenThrow(mock(JetStreamApiException.class)); + assertThrows( + RqueueNatsException.class, + () -> f.broker.enqueue( + queueNamed("orders"), RqueueMessage.builder().id("m1").message("hi").build())); + } + + @Test + void publish_writesUtf8BytesToConnection() { + Fixture f = newFixture(RqueueNatsConfig.defaults()); + f.broker.publish("chan-1", "hello"); + verify(f.conn, times(1)).publish("chan-1", "hello".getBytes(UTF_8)); + } + + @Test + void subscribe_createsDispatcherAndSubscribesChannel_closeReleasesIt() throws Exception { + Fixture f = newFixture(RqueueNatsConfig.defaults()); + Dispatcher d = mock(Dispatcher.class); + when(f.conn.createDispatcher(any(MessageHandler.class))).thenReturn(d); + when(d.subscribe(any(String.class))).thenReturn(d); + + AutoCloseable closer = f.broker.subscribe("chan-1", payload -> {}); + verify(f.conn, times(1)).createDispatcher(any(MessageHandler.class)); + verify(d, times(1)).subscribe("chan-1"); + verify(f.conn, never()).closeDispatcher(any()); + + closer.close(); + verify(f.conn, times(1)).closeDispatcher(d); + } + + @Test + void enqueueReactive_completesWhenPublishFutureCompletes() { + Fixture f = newFixture(RqueueNatsConfig.defaults()); + PublishAck ack = mock(PublishAck.class); + CompletableFuture done = CompletableFuture.completedFuture(ack); + when(f.js.publishAsync(any(String.class), any(Headers.class), any(byte[].class))) + .thenReturn(done); + + StepVerifier.create(f.broker.enqueueReactive( + queueNamed("orders"), RqueueMessage.builder().id("m1").message("hi").build())) + .verifyComplete(); + verify(f.js, times(1)) + .publishAsync(eq("rqueue.js.orders"), any(Headers.class), any(byte[].class)); + } + + @Test + void enqueueReactive_wrapsAsyncFailureInRqueueNatsException() { + Fixture f = newFixture(RqueueNatsConfig.defaults()); + CompletableFuture failed = new CompletableFuture<>(); + failed.completeExceptionally(new IOException("network down")); + when(f.js.publishAsync(any(String.class), any(Headers.class), any(byte[].class))) + .thenReturn(failed); + + StepVerifier.create(f.broker.enqueueReactive( + queueNamed("orders"), RqueueMessage.builder().id("m1").message("hi").build())) + .expectError(RqueueNatsException.class) + .verify(); + } + + @Test + void enqueueWithDelayReactive_returnsErrorMonoOfUOE() { + Fixture f = newFixture(RqueueNatsConfig.defaults()); + StepVerifier.create(f.broker.enqueueWithDelayReactive( + queueNamed("orders"), RqueueMessage.builder().id("m1").message("hi").build(), 100)) + .expectError(UnsupportedOperationException.class) + .verify(); + } + + // ---- helper ----------------------------------------------------------- + + private static final class Fixture { + final Connection conn; + final JetStream js; + final JetStreamManagement jsm; + final JetStreamMessageBroker broker; + + Fixture(Connection conn, JetStream js, JetStreamManagement jsm, JetStreamMessageBroker broker) { + this.conn = conn; + this.js = js; + this.jsm = jsm; + this.broker = broker; + } + } +} diff --git a/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamQueueModeIT.java b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamQueueModeIT.java new file mode 100644 index 000000000..fc9917a0c --- /dev/null +++ b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/JetStreamQueueModeIT.java @@ -0,0 +1,234 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ +package com.github.sonus21.rqueue.nats; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.github.sonus21.rqueue.core.RqueueMessage; +import com.github.sonus21.rqueue.enums.QueueType; +import com.github.sonus21.rqueue.listener.QueueDetail; +import com.github.sonus21.rqueue.nats.internal.NatsProvisioner; +import com.github.sonus21.rqueue.nats.js.JetStreamMessageBroker; +import io.nats.client.JetStreamManagement; +import io.nats.client.api.ConsumerInfo; +import io.nats.client.api.RetentionPolicy; +import java.time.Duration; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import org.junit.jupiter.api.Test; + +/** + * E2E contracts for {@link QueueType}: + * + *

    + *
  1. Stream retention matches the declared mode (QUEUE → WorkQueue, STREAM → Limits). + *
  2. QUEUE mode reuses the same durable consumer across repeated {@code ensureConsumer} calls + * so message position is preserved rather than reset. + *
  3. QUEUE mode delivers each message to exactly one competing consumer. + *
  4. STREAM mode delivers every message to every independent consumer group (fan-out). + *
+ */ +@NatsIntegrationTest +class JetStreamQueueModeIT extends AbstractJetStreamIT { + + // ---- Contract 1: stream retention reflects QueueType ------------------ + + @Test + void queueMode_queue_createsWorkQueueStream() throws Exception { + String streamName = "rqueue-" + "qm-queue-" + System.nanoTime(); + String subject = "rqueue." + "qm-queue-" + System.nanoTime(); + JetStreamManagement jsm = connection.jetStreamManagement(); + NatsProvisioner provisioner = new NatsProvisioner(connection, jsm, RqueueNatsConfig.defaults()); + + provisioner.ensureStream(streamName, List.of(subject), QueueType.QUEUE); + + RetentionPolicy actual = jsm.getStreamInfo(streamName).getConfiguration().getRetentionPolicy(); + assertEquals( + RetentionPolicy.WorkQueue, actual, "QUEUE mode must create a WorkQueue-retention stream"); + } + + @Test + void queueMode_stream_createsLimitsStream() throws Exception { + String streamName = "rqueue-" + "qm-stream-" + System.nanoTime(); + String subject = "rqueue." + "qm-stream-" + System.nanoTime(); + JetStreamManagement jsm = connection.jetStreamManagement(); + NatsProvisioner provisioner = new NatsProvisioner(connection, jsm, RqueueNatsConfig.defaults()); + + provisioner.ensureStream(streamName, List.of(subject), QueueType.STREAM); + + RetentionPolicy actual = jsm.getStreamInfo(streamName).getConfiguration().getRetentionPolicy(); + assertEquals( + RetentionPolicy.Limits, actual, "STREAM mode must create a Limits-retention stream"); + } + + // ---- Contract 2: consumer reuse preserves delivery position ----------- + + /** + * Verifies that calling {@code ensureConsumer} twice with identical arguments does NOT reset the + * consumer's delivery position. If a new consumer were created each time (with + * {@code DeliverPolicy.All}), previously-acked messages would be redelivered. + * + *

Sequence: + *

    + *
  1. Enqueue 5 messages. + *
  2. Pop and ack 3 via consumer "c1". + *
  3. Call {@code ensureConsumer} again — simulates an app restart or a second bean init. + *
  4. Pop remaining — must yield exactly 2 (not 5 again). + *
+ */ + @Test + void queueMode_consumerReuse_preservesDeliveryPosition() throws Exception { + QueueDetail q = mockQueue("qm-reuse-" + System.nanoTime(), QueueType.QUEUE); + String consumerName = "c1-reuse"; + int total = 5; + int firstBatch = 3; + + try (JetStreamMessageBroker broker = + JetStreamMessageBroker.builder().connection(connection).build()) { + for (int i = 0; i < total; i++) { + broker.enqueue(q, RqueueMessage.builder().id("r-" + i).message("p" + i).build()); + } + + // Pop and ack the first batch. + Set firstSeen = new HashSet<>(); + long deadline = System.currentTimeMillis() + 10_000; + while (firstSeen.size() < firstBatch && System.currentTimeMillis() < deadline) { + List msgs = broker.pop(q, consumerName, 5, Duration.ofMillis(500)); + for (RqueueMessage m : msgs) { + if (firstSeen.add(m.getId())) { + broker.ack(q, m); + } + if (firstSeen.size() == firstBatch) { + break; + } + } + } + assertEquals(firstBatch, firstSeen.size(), "should have consumed the first batch"); + + // Simulate a second call to ensureConsumer (e.g. from NatsStreamValidator on restart). + // The provisioner cache is already warm so this is effectively a no-op on the server side, + // but we also test against a fresh provisioner to simulate a true restart scenario. + JetStreamManagement jsm = connection.jetStreamManagement(); + NatsProvisioner freshProvisioner = + new NatsProvisioner(connection, jsm, RqueueNatsConfig.defaults()); + RqueueNatsConfig.ConsumerDefaults cd = RqueueNatsConfig.defaults().getConsumerDefaults(); + freshProvisioner.ensureConsumer( + RqueueNatsConfig.defaults().getStreamPrefix() + q.getName(), + consumerName, + cd.getAckWait(), + cd.getMaxDeliver(), + cd.getMaxAckPending()); + + // Verify the consumer info still reflects the already-delivered messages. + ConsumerInfo info = jsm.getConsumerInfo( + RqueueNatsConfig.defaults().getStreamPrefix() + q.getName(), consumerName); + long numAcked = info.getNumAckPending() == 0 + ? total - info.getNumPending() + : total - info.getNumPending() - info.getNumAckPending(); + // At minimum, the pending count must not have reset to the full total. + long remaining = info.getNumPending() + info.getNumAckPending(); + assertEquals( + total - firstBatch, + remaining, + "consumer position must be preserved across ensureConsumer calls; " + "remaining=" + + remaining + " but expected " + (total - firstBatch)); + } + } + + // ---- Contract 3: QUEUE competing consumers — each message once -------- + + @Test + void queueMode_queue_competingConsumers_eachMessageDeliveredOnce() throws Exception { + QueueDetail q = mockQueue("qm-cc-" + System.nanoTime(), QueueType.QUEUE); + int total = 20; + + try (JetStreamMessageBroker broker = + JetStreamMessageBroker.builder().connection(connection).build()) { + for (int i = 0; i < total; i++) { + broker.enqueue(q, RqueueMessage.builder().id("cc-" + i).message("p" + i).build()); + } + + Set seen = ConcurrentHashMap.newKeySet(); + CountDownLatch done = new CountDownLatch(total); + String sharedConsumer = "shared-cc"; + var pool = Executors.newFixedThreadPool(2); + for (int t = 0; t < 2; t++) { + pool.submit(() -> { + for (int round = 0; round < 100 && done.getCount() > 0; round++) { + List msgs = broker.pop(q, sharedConsumer, 5, Duration.ofMillis(300)); + for (RqueueMessage m : msgs) { + if (seen.add(m.getId())) { + done.countDown(); + } + broker.ack(q, m); + } + } + }); + } + done.await(20, java.util.concurrent.TimeUnit.SECONDS); + pool.shutdownNow(); + + assertEquals( + total, seen.size(), "QUEUE mode: each message must be delivered to exactly one worker"); + } + } + + // ---- Contract 4: STREAM fan-out — every consumer sees every message --- + + @Test + void queueMode_stream_fanOut_everyConsumerReceivesAllMessages() throws Exception { + QueueDetail q = mockQueue("qm-fo-" + System.nanoTime(), QueueType.STREAM); + int total = 8; + + try (JetStreamMessageBroker broker = + JetStreamMessageBroker.builder().connection(connection).build()) { + for (int i = 0; i < total; i++) { + broker.enqueue(q, RqueueMessage.builder().id("fo-" + i).message("p" + i).build()); + } + + // Each listener group uses a distinct consumer name — they are independent on a + // Limits-retention stream and each track their own delivery position. + Set listenerOneSeen = drain(broker, q, "listener-svc-1", total); + Set listenerTwoSeen = drain(broker, q, "listener-svc-2", total); + + assertEquals( + total, listenerOneSeen.size(), "STREAM mode: listener-svc-1 must receive all messages"); + assertEquals( + total, + listenerTwoSeen.size(), + "STREAM mode: listener-svc-2 must receive all messages independently"); + } + } + + private Set drain( + JetStreamMessageBroker broker, QueueDetail q, String consumer, int expected) + throws InterruptedException { + Set seen = new HashSet<>(); + long deadline = System.currentTimeMillis() + 10_000; + while (seen.size() < expected && System.currentTimeMillis() < deadline) { + List msgs = broker.pop(q, consumer, expected, Duration.ofMillis(500)); + for (RqueueMessage m : msgs) { + seen.add(m.getId()); + broker.ack(q, m); + } + } + return seen; + } +} diff --git a/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/NatsIntegrationTest.java b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/NatsIntegrationTest.java new file mode 100644 index 000000000..d638c64a7 --- /dev/null +++ b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/NatsIntegrationTest.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ + +package com.github.sonus21.rqueue.nats; + +import com.github.sonus21.junit.TestTracerExtension; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.extension.ExtendWith; + +/** + * Meta-annotation for Docker-gated JetStream integration tests. Carries only the {@code nats} + * tag so the existing Redis-driven {@code integration_test} and {@code reactive_integration_test} + * jobs don't try to run them; they're picked up exclusively by the dedicated + * {@code nats_integration_test} CI job via {@code -DincludeTags=nats}. + */ +@Target({ElementType.TYPE}) +@Retention(RetentionPolicy.RUNTIME) +@Tag("nats") +@ExtendWith(TestTracerExtension.class) +public @interface NatsIntegrationTest {} diff --git a/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/NatsStreamValidatorProducerModeTest.java b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/NatsStreamValidatorProducerModeTest.java new file mode 100644 index 000000000..fc6afab99 --- /dev/null +++ b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/NatsStreamValidatorProducerModeTest.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2024-2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ +package com.github.sonus21.rqueue.nats; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.github.sonus21.rqueue.config.RqueueConfig; +import com.github.sonus21.rqueue.core.EndpointRegistry; +import com.github.sonus21.rqueue.enums.QueueType; +import com.github.sonus21.rqueue.listener.QueueDetail; +import com.github.sonus21.rqueue.models.enums.RqueueMode; +import com.github.sonus21.rqueue.nats.internal.NatsProvisioner; +import com.github.sonus21.rqueue.nats.js.NatsStreamValidator; +import java.time.Duration; +import java.util.Collections; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Unit-tests the producer-mode skip in {@link NatsStreamValidator}: streams must still be + * provisioned (publishers need them) but the per-queue durable consumer must NOT be created when + * the application is configured as a producer-only node. + */ +@NatsUnitTest +class NatsStreamValidatorProducerModeTest { + + private NatsProvisioner provisioner; + private RqueueNatsConfig natsConfig; + + private static QueueDetail queue(String name) { + return QueueDetail.builder() + .name(name) + .queueName(name) + .processingQueueName(name + "-pq") + .completedQueueName(name + "-cq") + .scheduledQueueName(name + "-sq") + .processingQueueChannelName(name + "-pch") + .scheduledQueueChannelName(name + "-sch") + .visibilityTimeout(30000) + .numRetry(3) + .priority(Collections.emptyMap()) + .active(true) + .type(QueueType.QUEUE) + .build(); + } + + @BeforeEach + void setUp() { + EndpointRegistry.delete(); + provisioner = mock(NatsProvisioner.class); + when(provisioner.ensureConsumer(anyString(), anyString(), any(), anyLong(), anyLong())) + .thenReturn("rqueue-consumer"); + natsConfig = RqueueNatsConfig.defaults(); + } + + @Test + void producerMode_skipsConsumerProvisioningButStillEnsuresStream() { + EndpointRegistry.register(queue("orders")); + RqueueConfig rqueueConfig = mock(RqueueConfig.class); + when(rqueueConfig.getMode()).thenReturn(RqueueMode.PRODUCER); + when(rqueueConfig.isProducer()).thenReturn(true); + + NatsStreamValidator validator = new NatsStreamValidator(provisioner, natsConfig, rqueueConfig); + validator.afterSingletonsInstantiated(); + + verify(provisioner, times(1)) + .ensureStream(eq(natsConfig.getStreamPrefix() + "orders"), any(), any(), any()); + verify(provisioner, never()) + .ensureConsumer(anyString(), anyString(), any(Duration.class), anyLong(), anyLong()); + } + + @Test + void consumerMode_provisionsBothStreamAndConsumer() { + EndpointRegistry.register(queue("orders")); + RqueueConfig rqueueConfig = mock(RqueueConfig.class); + when(rqueueConfig.getMode()).thenReturn(RqueueMode.BOTH); + + NatsStreamValidator validator = new NatsStreamValidator(provisioner, natsConfig, rqueueConfig); + validator.afterSingletonsInstantiated(); + + verify(provisioner, times(1)) + .ensureStream(eq(natsConfig.getStreamPrefix() + "orders"), any(), any(), any()); + verify(provisioner, times(1)) + .ensureConsumer( + eq(natsConfig.getStreamPrefix() + "orders"), + anyString(), + any(Duration.class), + anyLong(), + anyLong()); + } + + @Test + void nullRqueueConfig_treatedAsConsumerMode_provisionsBoth() { + EndpointRegistry.register(queue("orders")); + + NatsStreamValidator validator = new NatsStreamValidator(provisioner, natsConfig); + validator.afterSingletonsInstantiated(); + + verify(provisioner, times(1)) + .ensureConsumer(anyString(), anyString(), any(Duration.class), anyLong(), anyLong()); + } +} diff --git a/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/NatsUnitTest.java b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/NatsUnitTest.java new file mode 100644 index 000000000..f84f4e710 --- /dev/null +++ b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/NatsUnitTest.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ + +package com.github.sonus21.rqueue.nats; + +import com.github.sonus21.junit.TestTracerExtension; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.extension.ExtendWith; + +/** + * NATS unit tests; tagged so CI's {@code unit_test} job picks them up. Intentionally does not + * include {@code MockitoExtension} — the existing tests use {@code Mockito.mock()} directly and + * adding the extension would activate strict-stubbing and flag setup-only stubs as unnecessary. + */ +@Target({ElementType.TYPE}) +@Retention(RetentionPolicy.RUNTIME) +@Tag("unit") +@Tag("nats") +@ExtendWith(TestTracerExtension.class) +public @interface NatsUnitTest {} diff --git a/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/RqueueNatsConfigTest.java b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/RqueueNatsConfigTest.java new file mode 100644 index 000000000..e32600bac --- /dev/null +++ b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/RqueueNatsConfigTest.java @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ +package com.github.sonus21.rqueue.nats; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.github.sonus21.rqueue.nats.RqueueNatsConfig.ConsumerDefaults; +import com.github.sonus21.rqueue.nats.RqueueNatsConfig.StreamDefaults; +import io.nats.client.api.RetentionPolicy; +import io.nats.client.api.StorageType; +import java.time.Duration; +import org.junit.jupiter.api.Test; + +/** POJO-only tests for {@link RqueueNatsConfig}; no broker / no NATS. */ +@NatsUnitTest +class RqueueNatsConfigTest { + + @Test + void defaults_returnsSensibleValues() { + RqueueNatsConfig c = RqueueNatsConfig.defaults(); + assertNotNull(c.getStreamPrefix()); + assertNotNull(c.getSubjectPrefix()); + assertNotNull(c.getDlqStreamSuffix()); + assertNotNull(c.getDlqSubjectSuffix()); + assertTrue(c.isAutoCreateStreams()); + assertTrue(c.isAutoCreateConsumers()); + assertFalse(c.isAutoCreateDlqStream()); + assertNotNull(c.getDefaultFetchWait()); + assertTrue(c.getDefaultFetchWait().toMillis() > 0); + assertNotNull(c.getStreamDefaults()); + assertNotNull(c.getConsumerDefaults()); + // sanity-check sane consumer defaults + assertTrue(c.getConsumerDefaults().getMaxAckPending() > 0); + assertTrue(c.getConsumerDefaults().getMaxDeliver() > 0); + assertNotNull(c.getConsumerDefaults().getAckWait()); + } + + @Test + void fluentSetters_returnSameInstanceAndUpdateFields() { + RqueueNatsConfig c = RqueueNatsConfig.defaults(); + assertSame(c, c.setStreamPrefix("s-")); + assertSame(c, c.setSubjectPrefix("sub.")); + assertSame(c, c.setDlqStreamSuffix("-x")); + assertSame(c, c.setDlqSubjectSuffix(".x")); + assertSame(c, c.setAutoCreateStreams(false)); + assertSame(c, c.setAutoCreateConsumers(false)); + assertSame(c, c.setAutoCreateDlqStream(false)); + assertSame(c, c.setDefaultFetchWait(Duration.ofSeconds(7))); + + assertEquals("s-", c.getStreamPrefix()); + assertEquals("sub.", c.getSubjectPrefix()); + assertEquals("-x", c.getDlqStreamSuffix()); + assertEquals(".x", c.getDlqSubjectSuffix()); + assertEquals(false, c.isAutoCreateStreams()); + assertEquals(false, c.isAutoCreateConsumers()); + assertEquals(false, c.isAutoCreateDlqStream()); + assertEquals(Duration.ofSeconds(7), c.getDefaultFetchWait()); + } + + @Test + void streamDefaults_setNestedReference() { + RqueueNatsConfig c = RqueueNatsConfig.defaults(); + StreamDefaults sd = new StreamDefaults(); + assertSame(c, c.setStreamDefaults(sd)); + assertSame(sd, c.getStreamDefaults()); + } + + @Test + void consumerDefaults_setNestedReference() { + RqueueNatsConfig c = RqueueNatsConfig.defaults(); + ConsumerDefaults cd = new ConsumerDefaults(); + assertSame(c, c.setConsumerDefaults(cd)); + assertSame(cd, c.getConsumerDefaults()); + } + + @Test + void streamDefaults_fluentSettersRoundtripEachProperty() { + StreamDefaults sd = new StreamDefaults(); + assertSame(sd, sd.setReplicas(3)); + assertSame(sd, sd.setStorage(StorageType.Memory)); + assertSame(sd, sd.setRetention(RetentionPolicy.Limits)); + assertSame(sd, sd.setDuplicateWindow(Duration.ofMinutes(10))); + assertSame(sd, sd.setMaxMsgs(1234L)); + assertSame(sd, sd.setMaxBytes(99999L)); + + assertEquals(3, sd.getReplicas()); + assertEquals(StorageType.Memory, sd.getStorage()); + assertEquals(RetentionPolicy.Limits, sd.getRetention()); + assertEquals(Duration.ofMinutes(10), sd.getDuplicateWindow()); + assertEquals(1234L, sd.getMaxMsgs()); + assertEquals(99999L, sd.getMaxBytes()); + } + + @Test + void consumerDefaults_fluentSettersRoundtripEachProperty() { + ConsumerDefaults cd = new ConsumerDefaults(); + assertSame(cd, cd.setAckWait(Duration.ofSeconds(45))); + assertSame(cd, cd.setMaxDeliver(11)); + assertSame(cd, cd.setMaxAckPending(2048)); + + assertEquals(Duration.ofSeconds(45), cd.getAckWait()); + assertEquals(11L, cd.getMaxDeliver()); + assertEquals(2048L, cd.getMaxAckPending()); + } + + @Test + void emptyAndNullPrefixesAreAccepted() { + RqueueNatsConfig c = RqueueNatsConfig.defaults().setSubjectPrefix("").setStreamPrefix(null); + assertEquals("", c.getSubjectPrefix()); + assertEquals(null, c.getStreamPrefix()); + } + + @Test + void toString_doesNotNpeOnDefaults() { + // RqueueNatsConfig is plain Java (no Lombok) — but still verify no surprise NPEs and equals to + // self holds. + RqueueNatsConfig c = RqueueNatsConfig.defaults(); + assertDoesNotThrow(c::toString); + assertEquals(c, c); + } +} diff --git a/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/dao/NatsRqueueJobDaoIT.java b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/dao/NatsRqueueJobDaoIT.java new file mode 100644 index 000000000..7c916f087 --- /dev/null +++ b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/dao/NatsRqueueJobDaoIT.java @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ +package com.github.sonus21.rqueue.nats.dao; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.github.sonus21.rqueue.models.db.RqueueJob; +import com.github.sonus21.rqueue.models.enums.JobStatus; +import com.github.sonus21.rqueue.nats.AbstractJetStreamIT; +import com.github.sonus21.rqueue.nats.RqueueNatsConfig; +import com.github.sonus21.rqueue.nats.internal.NatsProvisioner; +import com.github.sonus21.rqueue.serdes.RqJacksonSerDes; +import com.github.sonus21.rqueue.serdes.SerializationUtils; +import io.nats.client.JetStreamApiException; +import io.nats.client.KeyValueManagement; +import java.io.IOException; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** Round-trip exercise for {@link NatsRqueueJobDao} against a real JetStream KV. */ +class NatsRqueueJobDaoIT extends AbstractJetStreamIT { + + private NatsRqueueJobDao dao; + + @BeforeEach + void freshBucket() throws IOException, JetStreamApiException { + KeyValueManagement kvm = connection.keyValueManagement(); + try { + kvm.delete("rqueue-jobs"); + } catch (JetStreamApiException notFound) { + // first run + } + NatsProvisioner provisioner = new NatsProvisioner( + connection, connection.jetStreamManagement(), RqueueNatsConfig.defaults()); + dao = new NatsRqueueJobDao( + provisioner, new RqJacksonSerDes(SerializationUtils.getObjectMapper())); + } + + private RqueueJob job(String id, String messageId) { + RqueueJob j = new RqueueJob(); + j.setId(id); + j.setMessageId(messageId); + j.setStatus(JobStatus.CREATED); + j.setCreatedAt(System.currentTimeMillis()); + return j; + } + + @Test + void saveAndFindById() { + RqueueJob j = job("j1", "m1"); + dao.save(j, Duration.ofMinutes(1)); + + RqueueJob back = dao.findById("j1"); + assertNotNull(back); + assertEquals("j1", back.getId()); + assertEquals("m1", back.getMessageId()); + assertEquals(JobStatus.CREATED, back.getStatus()); + } + + @Test + void findByIdMissingReturnsNull() { + assertNull(dao.findById("never-saved")); + } + + @Test + void findJobsByIdIn() { + dao.save(job("a", "ma"), Duration.ofMinutes(1)); + dao.save(job("b", "mb"), Duration.ofMinutes(1)); + dao.save(job("c", "mc"), Duration.ofMinutes(1)); + List got = dao.findJobsByIdIn(Arrays.asList("a", "b", "missing")); + assertEquals(2, got.size()); + } + + @Test + void findByMessageId() { + dao.save(job("j1", "msg-X"), Duration.ofMinutes(1)); + dao.save(job("j2", "msg-X"), Duration.ofMinutes(1)); // same message, different attempt + dao.save(job("j3", "msg-Y"), Duration.ofMinutes(1)); + List jobsForX = dao.finByMessageId("msg-X"); + assertEquals(2, jobsForX.size()); + assertTrue(jobsForX.stream().allMatch(j -> "msg-X".equals(j.getMessageId()))); + } + + @Test + void findByMessageIdIn() { + dao.save(job("j1", "m1"), Duration.ofMinutes(1)); + dao.save(job("j2", "m2"), Duration.ofMinutes(1)); + dao.save(job("j3", "m3"), Duration.ofMinutes(1)); + List got = dao.finByMessageIdIn(Arrays.asList("m1", "m3")); + assertEquals(2, got.size()); + } + + @Test + void deleteRemovesEntry() { + dao.save(job("to-delete", "mx"), Duration.ofMinutes(1)); + assertNotNull(dao.findById("to-delete")); + dao.delete("to-delete"); + assertNull(dao.findById("to-delete")); + } + + @Test + void sanitizationLetsIllegalIdsRoundTrip() { + RqueueJob j = job("job#with$weird:chars", "msg-1"); + dao.save(j, Duration.ofMinutes(1)); + RqueueJob back = dao.findById("job#with$weird:chars"); + assertNotNull(back); + assertEquals("msg-1", back.getMessageId()); + } +} diff --git a/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/dao/NatsRqueueSystemConfigDaoIT.java b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/dao/NatsRqueueSystemConfigDaoIT.java new file mode 100644 index 000000000..ce677513f --- /dev/null +++ b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/dao/NatsRqueueSystemConfigDaoIT.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ +package com.github.sonus21.rqueue.nats.dao; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import com.github.sonus21.rqueue.models.db.QueueConfig; +import com.github.sonus21.rqueue.nats.AbstractJetStreamIT; +import com.github.sonus21.rqueue.nats.RqueueNatsConfig; +import com.github.sonus21.rqueue.nats.internal.NatsProvisioner; +import com.github.sonus21.rqueue.serdes.RqJacksonSerDes; +import com.github.sonus21.rqueue.serdes.SerializationUtils; +import io.nats.client.JetStreamApiException; +import io.nats.client.KeyValueManagement; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** Round-trip exercise for {@link NatsRqueueSystemConfigDao} against a real JetStream KV. */ +class NatsRqueueSystemConfigDaoIT extends AbstractJetStreamIT { + + private NatsRqueueSystemConfigDao dao; + + @BeforeEach + void freshBucket() throws IOException, JetStreamApiException { + KeyValueManagement kvm = connection.keyValueManagement(); + try { + kvm.delete("rqueue-queue-config"); + } catch (JetStreamApiException notFound) { + // first run + } + NatsProvisioner provisioner = new NatsProvisioner( + connection, connection.jetStreamManagement(), RqueueNatsConfig.defaults()); + dao = new NatsRqueueSystemConfigDao( + provisioner, new RqJacksonSerDes(SerializationUtils.getObjectMapper())); + } + + private QueueConfig sample(String id, String name) { + QueueConfig c = new QueueConfig(); + c.setId(id); + c.setName(name); + c.setQueueName(name); + c.setNumRetry(3); + c.setVisibilityTimeout(30_000L); + return c; + } + + @Test + void saveAndGetByName() { + QueueConfig c = sample("id-1", "orders"); + dao.saveQConfig(c); + + QueueConfig back = dao.getConfigByName("orders", false); + assertNotNull(back); + assertEquals("id-1", back.getId()); + assertEquals("orders", back.getName()); + assertEquals(3, back.getNumRetry()); + assertEquals(30_000L, back.getVisibilityTimeout()); + } + + @Test + void getByNameMissingReturnsNull() { + assertNull(dao.getConfigByName("never-saved", false)); + } + + @Test + void cachedReadDoesNotRoundtripJetStream() { + QueueConfig c = sample("id-2", "cached-q"); + dao.saveQConfig(c); + // First read primes the cache; second hits cache. + QueueConfig first = dao.getConfigByName("cached-q"); + QueueConfig second = dao.getConfigByName("cached-q"); + assertNotNull(first); + assertEquals(first, second); + + dao.clearCacheByName("cached-q"); + QueueConfig third = dao.getConfigByName("cached-q"); + assertNotNull(third); + assertEquals("cached-q", third.getName()); + } + + @Test + void saveAllAndGetByNames() { + dao.saveAllQConfig(Arrays.asList(sample("a", "qa"), sample("b", "qb"), sample("c", "qc"))); + List got = dao.getConfigByNames(Arrays.asList("qa", "qb", "missing")); + assertEquals(2, got.size()); + } + + @Test + void getQConfigByIdScan() { + dao.saveAllQConfig(Arrays.asList(sample("alpha", "q1"), sample("beta", "q2"))); + QueueConfig found = dao.getQConfig("beta", false); + assertNotNull(found); + assertEquals("q2", found.getName()); + } + + @Test + void sanitizationLetsNamesWithIllegalCharsRoundTrip() { + // Note: name is preserved on the QueueConfig itself; only the KV key is sanitized. + QueueConfig c = sample("id-x", "queue#with$weird:chars"); + dao.saveQConfig(c); + QueueConfig back = dao.getConfigByName("queue#with$weird:chars", false); + assertNotNull(back); + assertEquals("id-x", back.getId()); + } +} diff --git a/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/js/JetStreamMessageBrokerResolveTest.java b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/js/JetStreamMessageBrokerResolveTest.java new file mode 100644 index 000000000..5f92b69ef --- /dev/null +++ b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/js/JetStreamMessageBrokerResolveTest.java @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2024-2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ +package com.github.sonus21.rqueue.nats.js; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.github.sonus21.rqueue.listener.QueueDetail; +import com.github.sonus21.rqueue.nats.NatsUnitTest; +import com.github.sonus21.rqueue.nats.RqueueNatsConfig; +import java.time.Duration; +import org.junit.jupiter.api.Test; + +/** + * Pure-Java coverage for the static resolvers that translate rqueue's per-queue settings + * (visibilityTimeout, numRetry) into JetStream consumer config. These are the only place + * where backend semantics are mapped, so a regression here silently changes redelivery + * behaviour for every NATS-backed queue. + */ +@NatsUnitTest +class JetStreamMessageBrokerResolveTest { + + private static QueueDetail queue(long visibilityTimeoutMs, int numRetry) { + QueueDetail q = mock(QueueDetail.class); + when(q.getVisibilityTimeout()).thenReturn(visibilityTimeoutMs); + when(q.getNumRetry()).thenReturn(numRetry); + return q; + } + + // ---- resolveAckWait ---------------------------------------------------- + + @Test + void resolveAckWait_usesVisibilityTimeoutWhenPositive() { + QueueDetail q = queue(45_000L, 3); + assertEquals( + Duration.ofMillis(45_000L), + JetStreamMessageBroker.resolveAckWait(q, RqueueNatsConfig.defaults())); + } + + @Test + void resolveAckWait_fallsBackToConfigDefaultWhenZero() { + QueueDetail q = queue(0L, 3); + assertEquals( + Duration.ofSeconds(30), + JetStreamMessageBroker.resolveAckWait(q, RqueueNatsConfig.defaults())); + } + + @Test + void resolveAckWait_fallsBackToConfigDefaultWhenNegative() { + QueueDetail q = queue(-1L, 3); + assertEquals( + Duration.ofSeconds(30), + JetStreamMessageBroker.resolveAckWait(q, RqueueNatsConfig.defaults())); + } + + // ---- resolveMaxDeliver ------------------------------------------------- + + @Test + void resolveMaxDeliver_isNumRetryPlusOne() { + QueueDetail q = queue(30_000L, 3); + // numRetry=3 means 1 initial attempt + 3 retries = 4 deliveries total + assertEquals(4L, JetStreamMessageBroker.resolveMaxDeliver(q, RqueueNatsConfig.defaults())); + } + + @Test + void resolveMaxDeliver_fallsBackToConfigDefaultWhenZero() { + QueueDetail q = queue(30_000L, 0); + // RqueueNatsConfig.defaults().consumerDefaults.maxDeliver = 3 + assertEquals(3L, JetStreamMessageBroker.resolveMaxDeliver(q, RqueueNatsConfig.defaults())); + } + + @Test + void resolveMaxDeliver_fallsBackToConfigDefaultWhenNegative() { + QueueDetail q = queue(30_000L, -5); + assertEquals(3L, JetStreamMessageBroker.resolveMaxDeliver(q, RqueueNatsConfig.defaults())); + } + + /** + * rqueue uses {@link Integer#MAX_VALUE} as the "retry forever" sentinel. JetStream's Java + * client treats any non-positive {@code maxDeliver} as unset (omits the field from the + * wire payload) which lets the server default to unlimited — so {@code -1} is the right + * value to hand to the builder for that case. + */ + @Test + void resolveMaxDeliver_retryForeverSentinelMapsToUnlimited() { + QueueDetail q = queue(30_000L, Integer.MAX_VALUE); + assertEquals(-1L, JetStreamMessageBroker.resolveMaxDeliver(q, RqueueNatsConfig.defaults())); + } +} diff --git a/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/lock/NatsRqueueLockManagerIT.java b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/lock/NatsRqueueLockManagerIT.java new file mode 100644 index 000000000..5dcf53322 --- /dev/null +++ b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/lock/NatsRqueueLockManagerIT.java @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ +package com.github.sonus21.rqueue.nats.lock; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.github.sonus21.rqueue.nats.AbstractJetStreamIT; +import com.github.sonus21.rqueue.nats.RqueueNatsConfig; +import com.github.sonus21.rqueue.nats.internal.NatsProvisioner; +import io.nats.client.JetStreamApiException; +import io.nats.client.KeyValue; +import io.nats.client.KeyValueManagement; +import java.io.IOException; +import java.time.Duration; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Exercises {@link NatsRqueueLockManager}'s NATS KV-backed acquire / release semantics. + * + *
    + *
  • First acquire wins; concurrent acquire of the same key by a different holder fails. + *
  • Release with a matching value succeeds; release with a different value is a no-op. + *
  • After a successful release the key is again acquirable by anyone. + *
+ */ +class NatsRqueueLockManagerIT extends AbstractJetStreamIT { + + private NatsRqueueLockManager lockManager; + + @BeforeEach + void freshBucket() throws IOException, JetStreamApiException { + KeyValueManagement kvm = connection.keyValueManagement(); + try { + kvm.delete("rqueue-locks"); + } catch (JetStreamApiException notFound) { + // bucket didn't exist; first run + } + NatsProvisioner provisioner = new NatsProvisioner( + connection, connection.jetStreamManagement(), RqueueNatsConfig.defaults()); + lockManager = new NatsRqueueLockManager(provisioner); + } + + @Test + void acquireWhenKeyAbsent() { + assertTrue(lockManager.acquireLock("k1", "holder-A", Duration.ofSeconds(10))); + } + + @Test + void acquireRejectsConcurrentHolder() { + assertTrue(lockManager.acquireLock("k2", "holder-A", Duration.ofSeconds(10))); + assertFalse(lockManager.acquireLock("k2", "holder-B", Duration.ofSeconds(10))); + } + + @Test + void releaseSucceedsForMatchingHolder() { + lockManager.acquireLock("k3", "holder-A", Duration.ofSeconds(10)); + assertTrue(lockManager.releaseLock("k3", "holder-A")); + // After release the key is acquirable again. + assertTrue(lockManager.acquireLock("k3", "holder-B", Duration.ofSeconds(10))); + } + + @Test + void releaseRejectsForeignHolder() throws IOException, JetStreamApiException { + lockManager.acquireLock("k4", "holder-A", Duration.ofSeconds(10)); + assertFalse(lockManager.releaseLock("k4", "holder-B")); + // Lock still held: holder-A's value remains in the bucket. + KeyValue kv = connection.keyValue("rqueue-locks"); + assertEquals("holder-A", new String(kv.get("k4").getValue())); + } + + @Test + void releaseOnAbsentKeyReturnsFalse() { + assertFalse(lockManager.releaseLock("never-acquired", "holder-A")); + } + + @Test + void sanitizationCoercesIllegalKeyCharacters() { + // The KV layer rejects '$', '#', etc.; lock manager sanitizes them transparently. + assertTrue( + lockManager.acquireLock("queue#$/with-illegal:chars", "holder-A", Duration.ofSeconds(10))); + } +} diff --git a/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/service/NatsRqueueMessageMetadataServiceIT.java b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/service/NatsRqueueMessageMetadataServiceIT.java new file mode 100644 index 000000000..70eaa7def --- /dev/null +++ b/rqueue-nats/src/test/java/com/github/sonus21/rqueue/nats/service/NatsRqueueMessageMetadataServiceIT.java @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + */ +package com.github.sonus21.rqueue.nats.service; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.github.sonus21.rqueue.core.RqueueMessage; +import com.github.sonus21.rqueue.core.support.RqueueMessageUtils; +import com.github.sonus21.rqueue.models.db.MessageMetadata; +import com.github.sonus21.rqueue.models.enums.MessageStatus; +import com.github.sonus21.rqueue.nats.AbstractJetStreamIT; +import com.github.sonus21.rqueue.nats.RqueueNatsConfig; +import com.github.sonus21.rqueue.nats.internal.NatsProvisioner; +import com.github.sonus21.rqueue.serdes.RqJacksonSerDes; +import com.github.sonus21.rqueue.serdes.SerializationUtils; +import io.nats.client.JetStreamApiException; +import io.nats.client.KeyValueManagement; +import java.io.IOException; +import java.time.Duration; +import java.util.Arrays; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** Round-trip exercise for {@link NatsRqueueMessageMetadataService} against a real JetStream KV. */ +class NatsRqueueMessageMetadataServiceIT extends AbstractJetStreamIT { + + private NatsRqueueMessageMetadataService svc; + + @BeforeEach + void freshBucket() throws IOException, JetStreamApiException { + KeyValueManagement kvm = connection.keyValueManagement(); + try { + kvm.delete("rqueue-message-metadata"); + } catch (JetStreamApiException notFound) { + // first run + } + NatsProvisioner provisioner = new NatsProvisioner( + connection, connection.jetStreamManagement(), RqueueNatsConfig.defaults()); + svc = new NatsRqueueMessageMetadataService( + provisioner, new RqJacksonSerDes(SerializationUtils.getObjectMapper())); + } + + private RqueueMessage rqueueMessage(String queue, String id) { + return RqueueMessage.builder().id(id).queueName(queue).message("payload").build(); + } + + private MessageMetadata metadata(String queue, String id, MessageStatus status) { + MessageMetadata m = new MessageMetadata(rqueueMessage(queue, id), status); + m.setUpdatedOn(System.currentTimeMillis()); + return m; + } + + @Test + void saveAndGetById() { + MessageMetadata m = metadata("orders", "msg-1", MessageStatus.ENQUEUED); + svc.save(m, Duration.ofMinutes(1), false); + MessageMetadata back = svc.get(m.getId()); + assertNotNull(back); + assertEquals(MessageStatus.ENQUEUED, back.getStatus()); + } + + @Test + void getByMessageIdLooksUpComputedMetaId() { + MessageMetadata m = metadata("orders", "msg-2", MessageStatus.ENQUEUED); + svc.save(m, Duration.ofMinutes(1), false); + MessageMetadata back = svc.getByMessageId("orders", "msg-2"); + assertNotNull(back); + assertEquals(RqueueMessageUtils.getMessageMetaId("orders", "msg-2"), back.getId()); + } + + @Test + void deleteRemovesEntry() { + MessageMetadata m = metadata("orders", "msg-3", MessageStatus.ENQUEUED); + svc.save(m, Duration.ofMinutes(1), false); + svc.delete(m.getId()); + assertNull(svc.get(m.getId())); + } + + @Test + void deleteMessageMarksDeletedFlag() { + MessageMetadata m = metadata("orders", "msg-4", MessageStatus.ENQUEUED); + svc.save(m, Duration.ofMinutes(1), false); + assertTrue(svc.deleteMessage("orders", "msg-4", Duration.ofMinutes(1))); + MessageMetadata back = svc.getByMessageId("orders", "msg-4"); + assertNotNull(back); + assertTrue(back.isDeleted()); + assertNotNull(back.getDeletedOn()); + } + + @Test + void deleteMessageOnMissingReturnsFalse() { + assertFalse(svc.deleteMessage("orders", "never-saved", Duration.ofMinutes(1))); + } + + @Test + void getOrCreateReturnsExistingWhenPresent() { + MessageMetadata seeded = metadata("orders", "msg-5", MessageStatus.ENQUEUED); + svc.save(seeded, Duration.ofMinutes(1), false); + MessageMetadata got = svc.getOrCreateMessageMetadata(rqueueMessage("orders", "msg-5")); + assertEquals(seeded.getId(), got.getId()); + } + + @Test + void getOrCreateReturnsNewWhenAbsent() { + MessageMetadata got = svc.getOrCreateMessageMetadata(rqueueMessage("orders", "fresh")); + assertNotNull(got); + assertEquals(MessageStatus.ENQUEUED, got.getStatus()); + } + + @Test + void findAllReturnsSavedSubset() { + MessageMetadata a = metadata("q", "a", MessageStatus.ENQUEUED); + MessageMetadata b = metadata("q", "b", MessageStatus.ENQUEUED); + svc.save(a, Duration.ofMinutes(1), false); + svc.save(b, Duration.ofMinutes(1), false); + var got = svc.findAll(Arrays.asList(a.getId(), b.getId(), "missing")); + assertEquals(2, got.size()); + } +} diff --git a/rqueue-redis/build.gradle b/rqueue-redis/build.gradle new file mode 100644 index 000000000..b94b2129e --- /dev/null +++ b/rqueue-redis/build.gradle @@ -0,0 +1,52 @@ +plugins { + id 'com.vanniktech.maven.publish' version '0.28.0' +} +apply from: "${rootDir}/gradle/packaging.gradle" +apply from: "${rootDir}/gradle/test-runner.gradle" +apply from: "${rootDir}/gradle/code-publish.gradle" + +import com.vanniktech.maven.publish.SonatypeHost; + +mavenPublishing { + publishToMavenCentral(SonatypeHost.CENTRAL_PORTAL) + signAllPublications() + pom { + name = "Rqueue Redis" + description = "Redis backend for Rqueue. Provides Redis-shaped DAOs, message templates, and broker implementations consumed by rqueue-spring and rqueue-spring-boot-starter when rqueue.backend=redis (the default)." + url = "https://github.com/sonus21/rqueue" + licenses { + license { + name = "Apache License 2.0" + url = "https://www.apache.org/licenses/LICENSE-2.0.txt" + } + } + developers { + developer { + id = "sonus21" + name = "Sonu Kumar" + email = "sonunitw12@gmail.com" + } + } + scm { + url = "https://github.com/sonus21/rqueue" + connection = "scm:git:git://github.com/sonus21/rqueue.git" + developerConnection = "scm:git:ssh://git@github.com:sonus21/rqueue.git" + } + issueManagement { + system = "GitHub" + url = "https://github.com/sonus21/rqueue/issues" + } + } +} + +dependencies { + // Broker-impl module mirroring rqueue-nats. spring-data-redis is inherited as `api` from + // the root subprojects block, so consumers of rqueue-redis transitively get the Redis SDK. + api project(":rqueue-core") + // Redis impls implement web service interfaces declared in rqueue-web (e.g. RqueueQDetailService). + // After the planned repository-interface refactor this dep can drop back to api(":rqueue-core"). + api project(":rqueue-web") + testImplementation project(":rqueue-test-util") + testImplementation "io.lettuce:lettuce-core:${lettuceVersion}" + testImplementation "io.projectreactor:reactor-test:${projectReactorReactorTestVersion}" +} diff --git a/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/config/RqueueRedisListenerConfig.java b/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/config/RqueueRedisListenerConfig.java new file mode 100644 index 000000000..9ed713f36 --- /dev/null +++ b/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/config/RqueueRedisListenerConfig.java @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ +package com.github.sonus21.rqueue.redis.config; + +import com.github.sonus21.rqueue.common.RqueueLockManager; +import com.github.sonus21.rqueue.common.RqueueRedisTemplate; +import com.github.sonus21.rqueue.config.RedisBackendCondition; +import com.github.sonus21.rqueue.config.RqueueConfig; +import com.github.sonus21.rqueue.core.RqueueBeanProvider; +import com.github.sonus21.rqueue.core.RqueueInternalPubSubChannel; +import com.github.sonus21.rqueue.core.RqueueMessageTemplate; +import com.github.sonus21.rqueue.core.RqueueRedisListenerContainerFactory; +import com.github.sonus21.rqueue.core.spi.MessageBroker; +import com.github.sonus21.rqueue.core.spi.redis.RedisMessageBroker; +import com.github.sonus21.rqueue.dao.RqueueStringDao; +import com.github.sonus21.rqueue.listener.RqueueMessageListenerContainer; +import com.github.sonus21.rqueue.metrics.RqueueQueueMetricsProvider; +import com.github.sonus21.rqueue.redis.core.ProcessingQueueMessageScheduler; +import com.github.sonus21.rqueue.redis.core.ScheduledQueueMessageScheduler; +import com.github.sonus21.rqueue.redis.dao.RqueueStringDaoImpl; +import com.github.sonus21.rqueue.redis.lock.RqueueRedisLock; +import com.github.sonus21.rqueue.redis.metrics.RedisRqueueQueueMetricsProvider; +import com.github.sonus21.rqueue.redis.repository.RedisMessageBrowsingRepository; +import com.github.sonus21.rqueue.redis.worker.RedisWorkerRegistryStore; +import com.github.sonus21.rqueue.repository.MessageBrowsingRepository; +import com.github.sonus21.rqueue.utils.RedisUtils; +import com.github.sonus21.rqueue.worker.RqueueWorkerRegistry; +import com.github.sonus21.rqueue.worker.RqueueWorkerRegistryImpl; +import com.github.sonus21.rqueue.worker.WorkerRegistryStore; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.redis.core.RedisTemplate; + +/** + * Redis-conditional bean wiring extracted from {@code RqueueListenerBaseConfig} so that the + * Redis-only impl classes can live in {@code rqueue-redis} without forcing {@code rqueue-core} to + * depend on this module. {@code RqueueListenerConfig} (non-Boot) and + * {@code RqueueListenerAutoConfig} (Boot) each {@code @Import} this configuration, so the Redis + * @Beans are registered exactly where they used to be. + * + *

Mirrors the role {@code RqueueNatsAutoConfig} plays for the NATS backend. + */ +@Configuration +@Conditional(RedisBackendCondition.class) +@ComponentScan({ + "com.github.sonus21.rqueue.redis", +}) +public class RqueueRedisListenerConfig { + + @Bean + @Conditional(RedisBackendCondition.class) + public MessageBroker redisMessageBroker(RqueueMessageTemplate rqueueMessageTemplate) { + return new RedisMessageBroker(rqueueMessageTemplate); + } + + @Bean + @Conditional(RedisBackendCondition.class) + public RqueueStringDao rqueueStringDao(RqueueConfig rqueueConfig) { + return new RqueueStringDaoImpl(rqueueConfig); + } + + @Bean + @Conditional(RedisBackendCondition.class) + public RedisTemplate rqueueRedisLongTemplate(RqueueConfig rqueueConfig) { + return RedisUtils.getRedisTemplate(rqueueConfig.getConnectionFactory()); + } + + @Bean + @Conditional(RedisBackendCondition.class) + public RqueueRedisListenerContainerFactory rqueueRedisListenerContainerFactory() { + return new RqueueRedisListenerContainerFactory(); + } + + @Bean + @Conditional(RedisBackendCondition.class) + public RqueueRedisTemplate stringRqueueRedisTemplate(RqueueConfig rqueueConfig) { + return new RqueueRedisTemplate<>(rqueueConfig.getConnectionFactory()); + } + + @Bean + @Conditional(RedisBackendCondition.class) + public RqueueInternalPubSubChannel rqueueInternalPubSubChannel( + RqueueRedisListenerContainerFactory rqueueRedisListenerContainerFactory, + RqueueMessageListenerContainer rqueueMessageListenerContainer, + RqueueConfig rqueueConfig, + RqueueBeanProvider rqueueBeanProvider, + @Qualifier("stringRqueueRedisTemplate") + RqueueRedisTemplate stringRqueueRedisTemplate) { + return new RqueueInternalPubSubChannel( + rqueueRedisListenerContainerFactory, + rqueueMessageListenerContainer, + rqueueConfig, + stringRqueueRedisTemplate, + rqueueBeanProvider); + } + + /** + * Pulls due delayed messages from the per-queue ZSET back onto the ready LIST. Redis-only; NATS + * uses JetStream's native redelivery instead. + */ + @Bean + @Conditional(RedisBackendCondition.class) + public ScheduledQueueMessageScheduler scheduledMessageScheduler() { + return new ScheduledQueueMessageScheduler(); + } + + /** + * Re-queues messages whose ack-window expired without explicit ack. Redis-only; the equivalent on + * NATS is the consumer's {@code AckWait} timer. + */ + @Bean + @Conditional(RedisBackendCondition.class) + public ProcessingQueueMessageScheduler processingMessageScheduler() { + return new ProcessingQueueMessageScheduler(); + } + + @Bean + @Conditional(RedisBackendCondition.class) + public WorkerRegistryStore redisWorkerRegistryStore(RqueueConfig rqueueConfig) { + return new RedisWorkerRegistryStore(rqueueConfig); + } + + @Bean + @Conditional(RedisBackendCondition.class) + public MessageBrowsingRepository messageBrowsingRepository( + @Qualifier("stringRqueueRedisTemplate") + RqueueRedisTemplate stringRqueueRedisTemplate) { + return new RedisMessageBrowsingRepository(stringRqueueRedisTemplate); + } + + @Bean + @Conditional(RedisBackendCondition.class) + public RqueueWorkerRegistry rqueueWorkerRegistry( + RqueueConfig rqueueConfig, WorkerRegistryStore workerRegistryStore) { + return new RqueueWorkerRegistryImpl(rqueueConfig, workerRegistryStore); + } + + @Bean + @Conditional(RedisBackendCondition.class) + public RqueueLockManager rqueueLockManager(RqueueStringDao rqueueStringDao) { + return new RqueueRedisLock(rqueueStringDao); + } + + /** + * Backend-agnostic queue-depth gauge source consumed by + * {@link com.github.sonus21.rqueue.metrics.RqueueMetrics}. Reuses the existing + * {@code stringRqueueRedisTemplate} bean so we don't add a new connection-bound dependency. + */ + @Bean + @Conditional(RedisBackendCondition.class) + public RqueueQueueMetricsProvider rqueueQueueMetricsProvider( + @Qualifier("stringRqueueRedisTemplate") + RqueueRedisTemplate stringRqueueRedisTemplate) { + return new RedisRqueueQueueMetricsProvider(stringRqueueRedisTemplate); + } +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/MessageScheduler.java b/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/core/MessageScheduler.java similarity index 97% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/core/MessageScheduler.java rename to rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/core/MessageScheduler.java index 83d20cc94..3e93f4ade 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/MessageScheduler.java +++ b/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/core/MessageScheduler.java @@ -14,13 +14,16 @@ * */ -package com.github.sonus21.rqueue.core; +package com.github.sonus21.rqueue.redis.core; import static java.lang.Math.min; import com.github.sonus21.rqueue.config.RqueueConfig; import com.github.sonus21.rqueue.config.RqueueSchedulerConfig; +import com.github.sonus21.rqueue.core.EndpointRegistry; +import com.github.sonus21.rqueue.core.RedisScriptFactory; import com.github.sonus21.rqueue.core.RedisScriptFactory.ScriptType; +import com.github.sonus21.rqueue.core.RqueueRedisListenerContainerFactory; import com.github.sonus21.rqueue.listener.QueueDetail; import com.github.sonus21.rqueue.models.event.RqueueBootstrapEvent; import com.github.sonus21.rqueue.utils.Constants; diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/ProcessingQueueMessageScheduler.java b/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/core/ProcessingQueueMessageScheduler.java similarity index 95% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/core/ProcessingQueueMessageScheduler.java rename to rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/core/ProcessingQueueMessageScheduler.java index e89035a86..e18f498c5 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/ProcessingQueueMessageScheduler.java +++ b/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/core/ProcessingQueueMessageScheduler.java @@ -14,10 +14,11 @@ * */ -package com.github.sonus21.rqueue.core; +package com.github.sonus21.rqueue.redis.core; import static java.lang.Long.max; +import com.github.sonus21.rqueue.core.EndpointRegistry; import com.github.sonus21.rqueue.listener.QueueDetail; import java.util.List; import java.util.Map; diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RedisScheduleTriggerHandler.java b/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/core/RedisScheduleTriggerHandler.java similarity index 98% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RedisScheduleTriggerHandler.java rename to rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/core/RedisScheduleTriggerHandler.java index ea94695d0..b6f953e56 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RedisScheduleTriggerHandler.java +++ b/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/core/RedisScheduleTriggerHandler.java @@ -13,9 +13,10 @@ * See the License for the specific language governing permissions and limitations under the License. * */ -package com.github.sonus21.rqueue.core; +package com.github.sonus21.rqueue.redis.core; import com.github.sonus21.rqueue.config.RqueueSchedulerConfig; +import com.github.sonus21.rqueue.core.RqueueRedisListenerContainerFactory; import com.github.sonus21.rqueue.utils.ThreadUtils; import com.google.common.annotations.VisibleForTesting; import java.util.HashMap; diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/ScheduledQueueMessageScheduler.java b/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/core/ScheduledQueueMessageScheduler.java similarity index 94% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/core/ScheduledQueueMessageScheduler.java rename to rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/core/ScheduledQueueMessageScheduler.java index beedce524..6aaabf0ba 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/ScheduledQueueMessageScheduler.java +++ b/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/core/ScheduledQueueMessageScheduler.java @@ -14,8 +14,9 @@ * */ -package com.github.sonus21.rqueue.core; +package com.github.sonus21.rqueue.redis.core; +import com.github.sonus21.rqueue.core.EndpointRegistry; import lombok.extern.slf4j.Slf4j; import org.slf4j.Logger; diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/dao/impl/RqueueJobDaoImpl.java b/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/dao/RqueueJobDaoImpl.java similarity index 94% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/dao/impl/RqueueJobDaoImpl.java rename to rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/dao/RqueueJobDaoImpl.java index 91500b86f..ff81a7e28 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/dao/impl/RqueueJobDaoImpl.java +++ b/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/dao/RqueueJobDaoImpl.java @@ -14,9 +14,10 @@ * */ -package com.github.sonus21.rqueue.dao.impl; +package com.github.sonus21.rqueue.redis.dao; import com.github.sonus21.rqueue.common.RqueueRedisTemplate; +import com.github.sonus21.rqueue.config.RedisBackendCondition; import com.github.sonus21.rqueue.config.RqueueConfig; import com.github.sonus21.rqueue.dao.RqueueJobDao; import com.github.sonus21.rqueue.dao.RqueueStringDao; @@ -30,8 +31,10 @@ import java.util.Objects; import java.util.stream.Collectors; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Conditional; import org.springframework.stereotype.Repository; +@Conditional(RedisBackendCondition.class) @Repository public class RqueueJobDaoImpl implements RqueueJobDao { diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/dao/impl/RqueueMessageMetadataDaoImpl.java b/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/dao/RqueueMessageMetadataDaoImpl.java similarity index 94% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/dao/impl/RqueueMessageMetadataDaoImpl.java rename to rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/dao/RqueueMessageMetadataDaoImpl.java index 1fa8e961a..154e2950f 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/dao/impl/RqueueMessageMetadataDaoImpl.java +++ b/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/dao/RqueueMessageMetadataDaoImpl.java @@ -14,10 +14,11 @@ * */ -package com.github.sonus21.rqueue.dao.impl; +package com.github.sonus21.rqueue.redis.dao; import com.github.sonus21.rqueue.common.ReactiveRqueueRedisTemplate; import com.github.sonus21.rqueue.common.RqueueRedisTemplate; +import com.github.sonus21.rqueue.config.RedisBackendCondition; import com.github.sonus21.rqueue.config.RqueueConfig; import com.github.sonus21.rqueue.dao.RqueueMessageMetadataDao; import com.github.sonus21.rqueue.exception.DuplicateMessageException; @@ -27,10 +28,12 @@ import java.util.List; import java.util.Objects; import java.util.stream.Collectors; +import org.springframework.context.annotation.Conditional; import org.springframework.stereotype.Repository; import org.springframework.util.Assert; import reactor.core.publisher.Mono; +@Conditional(RedisBackendCondition.class) @Repository public class RqueueMessageMetadataDaoImpl implements RqueueMessageMetadataDao { diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/dao/impl/RqueueQStatsDaoImpl.java b/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/dao/RqueueQStatsDaoImpl.java similarity index 91% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/dao/impl/RqueueQStatsDaoImpl.java rename to rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/dao/RqueueQStatsDaoImpl.java index 9f44e5178..2856474d6 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/dao/impl/RqueueQStatsDaoImpl.java +++ b/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/dao/RqueueQStatsDaoImpl.java @@ -14,9 +14,10 @@ * */ -package com.github.sonus21.rqueue.dao.impl; +package com.github.sonus21.rqueue.redis.dao; import com.github.sonus21.rqueue.common.RqueueRedisTemplate; +import com.github.sonus21.rqueue.config.RedisBackendCondition; import com.github.sonus21.rqueue.config.RqueueConfig; import com.github.sonus21.rqueue.dao.RqueueQStatsDao; import com.github.sonus21.rqueue.models.db.QueueStatistics; @@ -25,8 +26,10 @@ import java.util.Objects; import java.util.stream.Collectors; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Conditional; import org.springframework.stereotype.Repository; +@Conditional(RedisBackendCondition.class) @Repository public class RqueueQStatsDaoImpl implements RqueueQStatsDao { diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/dao/impl/RqueueStringDaoImpl.java b/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/dao/RqueueStringDaoImpl.java similarity index 99% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/dao/impl/RqueueStringDaoImpl.java rename to rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/dao/RqueueStringDaoImpl.java index e37354379..3b88bc449 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/dao/impl/RqueueStringDaoImpl.java +++ b/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/dao/RqueueStringDaoImpl.java @@ -14,7 +14,7 @@ * */ -package com.github.sonus21.rqueue.dao.impl; +package com.github.sonus21.rqueue.redis.dao; import com.github.sonus21.rqueue.common.RqueueRedisTemplate; import com.github.sonus21.rqueue.config.RqueueConfig; diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/dao/impl/RqueueSystemConfigDaoImpl.java b/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/dao/RqueueSystemConfigDaoImpl.java similarity index 95% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/dao/impl/RqueueSystemConfigDaoImpl.java rename to rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/dao/RqueueSystemConfigDaoImpl.java index 34a55d292..cd03f773f 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/dao/impl/RqueueSystemConfigDaoImpl.java +++ b/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/dao/RqueueSystemConfigDaoImpl.java @@ -14,9 +14,10 @@ * */ -package com.github.sonus21.rqueue.dao.impl; +package com.github.sonus21.rqueue.redis.dao; import com.github.sonus21.rqueue.common.RqueueRedisTemplate; +import com.github.sonus21.rqueue.config.RedisBackendCondition; import com.github.sonus21.rqueue.config.RqueueConfig; import com.github.sonus21.rqueue.dao.RqueueSystemConfigDao; import com.github.sonus21.rqueue.models.db.QueueConfig; @@ -29,9 +30,11 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Conditional; import org.springframework.stereotype.Repository; import org.springframework.util.CollectionUtils; +@Conditional(RedisBackendCondition.class) @Repository public class RqueueSystemConfigDaoImpl implements RqueueSystemConfigDao { diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/common/impl/RqueueLockManagerImpl.java b/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/lock/RqueueRedisLock.java similarity index 90% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/common/impl/RqueueLockManagerImpl.java rename to rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/lock/RqueueRedisLock.java index ae5d8888e..23a335dfe 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/common/impl/RqueueLockManagerImpl.java +++ b/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/lock/RqueueRedisLock.java @@ -14,7 +14,7 @@ * */ -package com.github.sonus21.rqueue.common.impl; +package com.github.sonus21.rqueue.redis.lock; import com.github.sonus21.rqueue.common.RqueueLockManager; import com.github.sonus21.rqueue.dao.RqueueStringDao; @@ -23,11 +23,11 @@ import org.springframework.util.Assert; @Slf4j -public class RqueueLockManagerImpl implements RqueueLockManager { +public class RqueueRedisLock implements RqueueLockManager { private final RqueueStringDao rqueueStringDao; - public RqueueLockManagerImpl(RqueueStringDao rqueueStringDao) { + public RqueueRedisLock(RqueueStringDao rqueueStringDao) { this.rqueueStringDao = rqueueStringDao; } diff --git a/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/metrics/RedisRqueueQueueMetricsProvider.java b/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/metrics/RedisRqueueQueueMetricsProvider.java new file mode 100644 index 000000000..245cc04b1 --- /dev/null +++ b/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/metrics/RedisRqueueQueueMetricsProvider.java @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ +package com.github.sonus21.rqueue.redis.metrics; + +import com.github.sonus21.rqueue.common.RqueueRedisTemplate; +import com.github.sonus21.rqueue.core.EndpointRegistry; +import com.github.sonus21.rqueue.exception.QueueDoesNotExist; +import com.github.sonus21.rqueue.listener.QueueDetail; +import com.github.sonus21.rqueue.metrics.RqueueQueueMetricsProvider; +import java.util.function.ToLongFunction; + +/** + * Redis-backed {@link RqueueQueueMetricsProvider}. Reads sizes off the same + * {@link RqueueRedisTemplate} the rest of the Redis backend uses: pending and DLQ live in + * Redis lists (LLEN); scheduled and in-flight live in sorted sets (ZCARD). + * + *

Unknown queues (i.e. names not present in {@link EndpointRegistry}) yield {@code 0} rather + * than propagating {@link QueueDoesNotExist}, so callers can use the values directly as gauge + * readings without guarding every lookup. + */ +public class RedisRqueueQueueMetricsProvider implements RqueueQueueMetricsProvider { + + private final RqueueRedisTemplate redisTemplate; + + public RedisRqueueQueueMetricsProvider(RqueueRedisTemplate redisTemplate) { + this.redisTemplate = redisTemplate; + } + + private long readSize(String queueName, ToLongFunction reader) { + try { + return reader.applyAsLong(EndpointRegistry.get(queueName)); + } catch (QueueDoesNotExist e) { + return 0L; + } + } + + private long readSize(String queueName, String priority, ToLongFunction reader) { + try { + return reader.applyAsLong(EndpointRegistry.get(queueName, priority)); + } catch (QueueDoesNotExist e) { + return 0L; + } + } + + private long listSize(String key) { + Long size = redisTemplate.getListSize(key); + return size == null ? 0L : size; + } + + private long zsetSize(String key) { + Long size = redisTemplate.getZsetSize(key); + return size == null ? 0L : size; + } + + @Override + public long getPendingMessageCount(String queueName) { + return readSize(queueName, q -> listSize(q.getQueueName())); + } + + @Override + public long getScheduledMessageCount(String queueName) { + return readSize(queueName, q -> zsetSize(q.getScheduledQueueName())); + } + + @Override + public long getProcessingMessageCount(String queueName) { + return readSize(queueName, q -> zsetSize(q.getProcessingQueueName())); + } + + @Override + public long getDeadLetterMessageCount(String queueName) { + return readSize(queueName, q -> q.isDlqSet() ? listSize(q.getDeadLetterQueueName()) : 0L); + } + + @Override + public long getPendingMessageCount(String queueName, String priority) { + return readSize(queueName, priority, q -> listSize(q.getQueueName())); + } + + @Override + public long getScheduledMessageCount(String queueName, String priority) { + return readSize(queueName, priority, q -> zsetSize(q.getScheduledQueueName())); + } + + @Override + public long getProcessingMessageCount(String queueName, String priority) { + return readSize(queueName, priority, q -> zsetSize(q.getProcessingQueueName())); + } +} diff --git a/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/repository/RedisMessageBrowsingRepository.java b/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/repository/RedisMessageBrowsingRepository.java new file mode 100644 index 000000000..13bfa3d4d --- /dev/null +++ b/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/repository/RedisMessageBrowsingRepository.java @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ + +package com.github.sonus21.rqueue.redis.repository; + +import com.github.sonus21.rqueue.common.RqueueRedisTemplate; +import com.github.sonus21.rqueue.exception.UnknownSwitchCase; +import com.github.sonus21.rqueue.models.enums.DataType; +import com.github.sonus21.rqueue.models.response.DataViewResponse; +import com.github.sonus21.rqueue.models.response.TableColumn; +import com.github.sonus21.rqueue.models.response.TableRow; +import com.github.sonus21.rqueue.repository.MessageBrowsingRepository; +import com.github.sonus21.rqueue.utils.RedisUtils; +import com.github.sonus21.rqueue.utils.StringUtils; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.springframework.data.redis.core.ZSetOperations.TypedTuple; + +/** + * Redis-backed {@link MessageBrowsingRepository}. Wraps {@link RqueueRedisTemplate} for the + * size and range primitives; delegates raw {@link org.springframework.data.redis.core.RedisTemplate} + * access for the bulk pipelining used by {@link #getDataSizes} (one round-trip for N queues + * instead of N). + * + *

This class exists to keep the Redis-shaped storage calls behind a stable interface so the + * single {@code RqueueQDetailServiceImpl} in {@code rqueue-web} can serve both backends. + */ +public class RedisMessageBrowsingRepository implements MessageBrowsingRepository { + + private final RqueueRedisTemplate stringTemplate; + + public RedisMessageBrowsingRepository(RqueueRedisTemplate stringTemplate) { + this.stringTemplate = stringTemplate; + } + + @Override + public long getDataSize(String name, DataType type) { + Long size; + switch (type) { + case LIST: + size = stringTemplate.getListSize(name); + break; + case ZSET: + size = stringTemplate.getZsetSize(name); + break; + default: + // SET / KEY sizes are not used by the dashboard; return 0 rather than throw. + return 0L; + } + return size == null ? 0L : size; + } + + @Override + public List getDataSizes(List names, List types) { + if (names == null || names.isEmpty()) { + return Collections.emptyList(); + } + if (types == null || names.size() != types.size()) { + throw new IllegalArgumentException("names and types must be the same length; names=" + + names.size() + " types=" + (types == null ? "null" : types.size())); + } + List raw = RedisUtils.executePipeLine( + stringTemplate.getRedisTemplate(), (connection, keySerializer, valueSerializer) -> { + for (int i = 0; i < names.size(); i++) { + byte[] key = keySerializer.serialize(names.get(i)); + switch (types.get(i)) { + case LIST: + connection.lLen(key); + break; + case ZSET: + connection.zCard(key); + break; + default: + // Unknown size: emit something to keep pipeline alignment with the input. + connection.exists(key); + } + } + }); + List out = new ArrayList<>(names.size()); + for (Object o : raw) { + if (o instanceof Number) { + out.add(((Number) o).longValue()); + } else { + out.add(0L); + } + } + return out; + } + + @Override + public DataViewResponse viewData( + String name, DataType type, String key, int pageNumber, int itemPerPage) { + switch (type) { + case SET: + return responseForSet(name); + case ZSET: + return responseForZset(name, key, pageNumber, itemPerPage); + case LIST: + return responseForList(name, pageNumber, itemPerPage); + case KEY: + return responseForKeyVal(name); + default: + throw new UnknownSwitchCase(type.name()); + } + } + + private DataViewResponse responseForSet(String name) { + List items = new ArrayList<>(stringTemplate.getMembers(name)); + DataViewResponse response = new DataViewResponse(); + response.setHeaders(Collections.singletonList("Item")); + List tableRows = new ArrayList<>(); + for (Object item : items) { + tableRows.add(new TableRow(new TableColumn(item.toString()))); + } + response.setRows(tableRows); + return response; + } + + private DataViewResponse responseForKeyVal(String name) { + DataViewResponse response = new DataViewResponse(); + response.setHeaders(Collections.singletonList("Value")); + Object val = stringTemplate.get(name); + response.addRow(new TableRow(new TableColumn(String.valueOf(val)))); + return response; + } + + private DataViewResponse responseForZset( + String name, String key, int pageNumber, int itemPerPage) { + DataViewResponse response = new DataViewResponse(); + int start = pageNumber * itemPerPage; + int end = start + itemPerPage - 1; + List tableRows = new ArrayList<>(); + if (!StringUtils.isEmpty(key)) { + Double score = stringTemplate.getZsetMemberScore(name, key); + response.setHeaders(Collections.singletonList("Score")); + tableRows.add(new TableRow(new TableColumn(score))); + } else { + response.setHeaders(Arrays.asList("Value", "Score")); + for (TypedTuple tuple : stringTemplate.zrangeWithScore(name, start, end)) { + tableRows.add(new TableRow(Arrays.asList( + new TableColumn(String.valueOf(tuple.getValue())), new TableColumn(tuple.getScore())))); + } + } + response.setRows(tableRows); + return response; + } + + private DataViewResponse responseForList(String name, int pageNumber, int itemPerPage) { + DataViewResponse response = new DataViewResponse(); + response.setHeaders(Collections.singletonList("Item")); + int start = pageNumber * itemPerPage; + int end = start + itemPerPage - 1; + List tableRows = new ArrayList<>(); + for (Object s : stringTemplate.lrange(name, start, end)) { + tableRows.add(new TableRow(new TableColumn(String.valueOf(s)))); + } + response.setRows(tableRows); + return response; + } +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueMessageMetadataServiceImpl.java b/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/web/RqueueMessageMetadataServiceImpl.java similarity index 94% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueMessageMetadataServiceImpl.java rename to rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/web/RqueueMessageMetadataServiceImpl.java index c86243c6e..94b0c151a 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueMessageMetadataServiceImpl.java +++ b/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/web/RqueueMessageMetadataServiceImpl.java @@ -14,7 +14,7 @@ * */ -package com.github.sonus21.rqueue.web.service.impl; +package com.github.sonus21.rqueue.redis.web; import com.github.sonus21.rqueue.common.RqueueLockManager; import com.github.sonus21.rqueue.config.RqueueConfig; @@ -24,8 +24,8 @@ import com.github.sonus21.rqueue.dao.RqueueStringDao; import com.github.sonus21.rqueue.models.db.MessageMetadata; import com.github.sonus21.rqueue.models.enums.MessageStatus; +import com.github.sonus21.rqueue.service.RqueueMessageMetadataService; import com.github.sonus21.rqueue.utils.Constants; -import com.github.sonus21.rqueue.web.service.RqueueMessageMetadataService; import java.time.Duration; import java.util.Collection; import java.util.Comparator; @@ -41,6 +41,12 @@ import org.springframework.stereotype.Service; import reactor.core.publisher.Mono; +/** + * Redis-shaped impl. Backend gating is handled at module level via + * {@code RqueueRedisListenerConfig.@Conditional(RedisBackendCondition)} + its + * {@code @ComponentScan} of {@code com.github.sonus21.rqueue.redis} — a per-class + * {@code @Conditional} would be redundant. + */ @Service @Slf4j public class RqueueMessageMetadataServiceImpl implements RqueueMessageMetadataService { diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueUtilityServiceImpl.java b/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/web/RqueueUtilityServiceImpl.java similarity index 93% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueUtilityServiceImpl.java rename to rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/web/RqueueUtilityServiceImpl.java index ccd770e77..9cc48b6de 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueUtilityServiceImpl.java +++ b/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/web/RqueueUtilityServiceImpl.java @@ -14,7 +14,7 @@ * */ -package com.github.sonus21.rqueue.web.service.impl; +package com.github.sonus21.rqueue.redis.web; import static com.github.sonus21.rqueue.utils.HttpUtils.readUrl; @@ -41,10 +41,10 @@ import com.github.sonus21.rqueue.models.response.DataSelectorResponse; import com.github.sonus21.rqueue.models.response.MessageMoveResponse; import com.github.sonus21.rqueue.models.response.StringResponse; +import com.github.sonus21.rqueue.service.RqueueMessageMetadataService; +import com.github.sonus21.rqueue.service.RqueueUtilityService; import com.github.sonus21.rqueue.utils.Constants; import com.github.sonus21.rqueue.utils.StringUtils; -import com.github.sonus21.rqueue.web.service.RqueueMessageMetadataService; -import com.github.sonus21.rqueue.web.service.RqueueUtilityService; import java.time.Duration; import java.util.LinkedHashMap; import java.util.LinkedList; @@ -55,6 +55,12 @@ import org.springframework.stereotype.Service; import reactor.core.publisher.Mono; +/** + * Redis-shaped impl. Backend gating is handled at module level via + * {@code RqueueRedisListenerConfig.@Conditional(RedisBackendCondition)} + its + * {@code @ComponentScan} of {@code com.github.sonus21.rqueue.redis} — a per-class + * {@code @Conditional} would be redundant. + */ @Service @Slf4j public class RqueueUtilityServiceImpl implements RqueueUtilityService { @@ -66,6 +72,13 @@ public class RqueueUtilityServiceImpl implements RqueueUtilityService { private final RqueueMessageMetadataService messageMetadataService; private final RqueueInternalPubSubChannel rqueueInternalPubSubChannel; private final RqueueConfig rqueueConfig; + /** + * Resolved once via {@link MessageSweeper#getInstance} at construction so {@link #makeEmpty} + * doesn't take the static-singleton path on every call. {@code getInstance} is itself a + * (Redis-shaped) singleton so this just pins it to a field for direct reuse. + */ + private final MessageSweeper messageSweeper; + private String latestVersion = "NA"; private String releaseLink = "#"; private long versionFetchTime = 0; @@ -86,6 +99,8 @@ public RqueueUtilityServiceImpl( this.rqueueMessageTemplate = rqueueMessageTemplate; this.messageMetadataService = messageMetadataService; this.rqueueInternalPubSubChannel = rqueueInternalPubSubChannel; + this.messageSweeper = + MessageSweeper.getInstance(rqueueConfig, rqueueMessageTemplate, messageMetadataService); } @Override @@ -208,13 +223,11 @@ public BooleanResponse makeEmpty(String queueName, String dataName) { if (type == null || type == org.springframework.data.redis.connection.DataType.NONE) { return new BooleanResponse(true); } - return new BooleanResponse( - MessageSweeper.getInstance(rqueueConfig, rqueueMessageTemplate, messageMetadataService) - .deleteAllMessages(MessageDeleteRequest.builder() - .dataName(dataName) - .queueName(queueName) - .dataType(type) - .build())); + return new BooleanResponse(messageSweeper.deleteAllMessages(MessageDeleteRequest.builder() + .dataName(dataName) + .queueName(queueName) + .dataType(type) + .build())); } private boolean shouldFetchVersionDetail() { diff --git a/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/worker/RedisWorkerRegistryStore.java b/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/worker/RedisWorkerRegistryStore.java new file mode 100644 index 000000000..734c2d5b8 --- /dev/null +++ b/rqueue-redis/src/main/java/com/github/sonus21/rqueue/redis/worker/RedisWorkerRegistryStore.java @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ + +package com.github.sonus21.rqueue.redis.worker; + +import com.github.sonus21.rqueue.common.RqueueRedisTemplate; +import com.github.sonus21.rqueue.config.RqueueConfig; +import com.github.sonus21.rqueue.models.registry.RqueueWorkerInfo; +import com.github.sonus21.rqueue.worker.WorkerRegistryStore; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.springframework.util.CollectionUtils; + +/** + * Redis-backed {@link WorkerRegistryStore}. Worker info lives at + * {@code rqueueConfig.getWorkerRegistryKey(workerId)} as a serialized {@link RqueueWorkerInfo}; + * per-queue heartbeats live in a Redis hash at + * {@code rqueueConfig.getWorkerRegistryQueueKey(queueName)} keyed by worker id with the JSON + * metadata payload as the value. + */ +public class RedisWorkerRegistryStore implements WorkerRegistryStore { + + private final RqueueRedisTemplate workerTemplate; + private final RqueueRedisTemplate stringTemplate; + + public RedisWorkerRegistryStore(RqueueConfig rqueueConfig) { + this.workerTemplate = new RqueueRedisTemplate<>(rqueueConfig.getConnectionFactory()); + this.stringTemplate = new RqueueRedisTemplate<>(rqueueConfig.getConnectionFactory()); + } + + @Override + public void putWorkerInfo(String workerKey, RqueueWorkerInfo info, Duration ttl) { + workerTemplate.set(workerKey, info, ttl); + } + + @Override + public void deleteWorkerInfo(String workerKey) { + workerTemplate.delete(workerKey); + } + + @Override + public Map getWorkerInfos(Collection workerKeys) { + if (CollectionUtils.isEmpty(workerKeys)) { + return Collections.emptyMap(); + } + List keys = new ArrayList<>(workerKeys); + List workerInfos = workerTemplate.mget(keys); + if (CollectionUtils.isEmpty(workerInfos)) { + return Collections.emptyMap(); + } + Map workerInfoById = new LinkedHashMap<>(); + for (RqueueWorkerInfo workerInfo : workerInfos) { + if (workerInfo != null && workerInfo.getWorkerId() != null) { + workerInfoById.put(workerInfo.getWorkerId(), workerInfo); + } + } + return workerInfoById; + } + + @Override + public void putQueueHeartbeat(String queueKey, String workerId, String metadataJson) { + stringTemplate.putHashValue(queueKey, workerId, metadataJson); + } + + @Override + public Map getQueueHeartbeats(String queueKey) { + Map entries = stringTemplate.getHashEntries(queueKey); + return entries == null ? Collections.emptyMap() : entries; + } + + @Override + public void deleteQueueHeartbeats(String queueKey, String... workerIds) { + if (workerIds == null || workerIds.length == 0) { + return; + } + stringTemplate.deleteHashValues(queueKey, workerIds); + } + + @Override + public void refreshQueueTtl(String queueKey, Duration ttl) { + stringTemplate.expire(queueKey, ttl); + } +} diff --git a/rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/RedisTestUtils.java b/rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/RedisTestUtils.java new file mode 100644 index 000000000..2e93e50bd --- /dev/null +++ b/rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/RedisTestUtils.java @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ + +package com.github.sonus21.rqueue.redis; + +import com.github.sonus21.rqueue.listener.QueueDetail; +import com.github.sonus21.rqueue.models.Concurrency; +import com.github.sonus21.rqueue.models.db.QueueConfig; +import java.util.HashMap; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; + +/** + * Module-local copy of the few helpers from {@code rqueue-core}'s test {@code TestUtils} that the + * relocated DAO-impl tests still need. Kept narrow on purpose; do not grow this without first + * checking whether the helper truly belongs in {@code rqueue-test-util} (shared) instead. + */ +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public final class RedisTestUtils { + + public static String getQueueConfigKey(String name) { + return "__rq::q-config::" + name; + } + + public static QueueConfig createQueueConfig(String name) { + return createQueueConfig(name, 3, 900000L, null); + } + + public static QueueConfig createQueueConfig( + String name, int numRetry, long visibilityTimeout, String dlq) { + QueueDetail detail = QueueDetail.builder() + .name(name) + .queueName("__rq::queue::" + name) + .processingQueueName("__rq::p-queue::" + name) + .processingQueueChannelName("__rq::p-channel::" + name) + .scheduledQueueName("__rq::d-queue::" + name) + .scheduledQueueChannelName("__rq::d-channel::" + name) + .completedQueueName("__rq::c-queue::" + name) + .numRetry(numRetry) + .visibilityTimeout(visibilityTimeout) + .deadLetterQueueName(dlq) + .priority(new HashMap<>()) + .priorityGroup("") + .concurrency(new Concurrency(-1, -1)) + .active(true) + .build(); + QueueConfig config = detail.toConfig(); + config.setId(getQueueConfigKey(name)); + return config; + } +} diff --git a/rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/RedisUnitTest.java b/rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/RedisUnitTest.java new file mode 100644 index 000000000..f63438a85 --- /dev/null +++ b/rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/RedisUnitTest.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ + +package com.github.sonus21.rqueue.redis; + +import com.github.sonus21.junit.TestTracerExtension; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + +/** + * Module-local mirror of {@code CoreUnitTest} from rqueue-core. Bundles the same JUnit tags and + * extensions so the DAO-impl tests that moved here keep their original wiring without + * cross-module test-source dependencies. + */ +@Target({ElementType.TYPE}) +@Retention(RetentionPolicy.RUNTIME) +@Tag("unit") +@Tag("redis") +@ExtendWith({MockitoExtension.class, TestTracerExtension.class}) +public @interface RedisUnitTest {} diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/common/RqueueLockManagerImplTest.java b/rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/common/RqueueRedisLockTest.java similarity index 89% rename from rqueue-core/src/test/java/com/github/sonus21/rqueue/common/RqueueLockManagerImplTest.java rename to rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/common/RqueueRedisLockTest.java index cc55f12cd..02c4802c1 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/common/RqueueLockManagerImplTest.java +++ b/rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/common/RqueueRedisLockTest.java @@ -14,7 +14,7 @@ * */ -package com.github.sonus21.rqueue.common; +package com.github.sonus21.rqueue.redis.common; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -22,8 +22,9 @@ import com.github.sonus21.TestBase; import com.github.sonus21.rqueue.CoreUnitTest; -import com.github.sonus21.rqueue.common.impl.RqueueLockManagerImpl; +import com.github.sonus21.rqueue.common.RqueueLockManager; import com.github.sonus21.rqueue.dao.RqueueStringDao; +import com.github.sonus21.rqueue.redis.lock.RqueueRedisLock; import java.time.Duration; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -31,7 +32,7 @@ import org.mockito.MockitoAnnotations; @CoreUnitTest -class RqueueLockManagerImplTest extends TestBase { +class RqueueRedisLockTest extends TestBase { private final String lockKey = "test-key"; private final String lockValue = "test-value"; @@ -44,7 +45,7 @@ class RqueueLockManagerImplTest extends TestBase { @BeforeEach public void init() { MockitoAnnotations.openMocks(this); - rqueueLockManager = new RqueueLockManagerImpl(rqueueStringDao); + rqueueLockManager = new RqueueRedisLock(rqueueStringDao); } @Test diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/MessageSchedulerDisabledTest.java b/rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/core/MessageSchedulerDisabledTest.java similarity index 97% rename from rqueue-core/src/test/java/com/github/sonus21/rqueue/core/MessageSchedulerDisabledTest.java rename to rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/core/MessageSchedulerDisabledTest.java index 822808bb8..27488f4fd 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/MessageSchedulerDisabledTest.java +++ b/rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/core/MessageSchedulerDisabledTest.java @@ -14,7 +14,7 @@ * */ -package com.github.sonus21.rqueue.core; +package com.github.sonus21.rqueue.redis.core; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; @@ -24,6 +24,7 @@ import com.github.sonus21.rqueue.CoreUnitTest; import com.github.sonus21.rqueue.config.RqueueConfig; import com.github.sonus21.rqueue.config.RqueueSchedulerConfig; +import com.github.sonus21.rqueue.core.EndpointRegistry; import com.github.sonus21.rqueue.listener.QueueDetail; import com.github.sonus21.rqueue.models.event.RqueueBootstrapEvent; import com.github.sonus21.rqueue.utils.TestUtils; diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/MessageSchedulerTest.java b/rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/core/MessageSchedulerTest.java similarity index 94% rename from rqueue-core/src/test/java/com/github/sonus21/rqueue/core/MessageSchedulerTest.java rename to rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/core/MessageSchedulerTest.java index 8f948a8f0..0d7bd1543 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/MessageSchedulerTest.java +++ b/rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/core/MessageSchedulerTest.java @@ -14,7 +14,7 @@ * */ -package com.github.sonus21.rqueue.core; +package com.github.sonus21.rqueue.redis.core; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.doReturn; @@ -23,9 +23,10 @@ import com.github.sonus21.rqueue.CoreUnitTest; import com.github.sonus21.rqueue.config.RqueueConfig; import com.github.sonus21.rqueue.config.RqueueSchedulerConfig; -import com.github.sonus21.rqueue.core.ScheduledQueueMessageSchedulerTest.TestScheduledQueueMessageScheduler; +import com.github.sonus21.rqueue.core.EndpointRegistry; import com.github.sonus21.rqueue.listener.QueueDetail; import com.github.sonus21.rqueue.models.event.RqueueBootstrapEvent; +import com.github.sonus21.rqueue.redis.core.ScheduledQueueMessageSchedulerTest.TestScheduledQueueMessageScheduler; import com.github.sonus21.rqueue.utils.TestUtils; import com.github.sonus21.test.TestTaskScheduler; import java.util.HashMap; diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/MessageSchedulingTest.java b/rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/core/MessageSchedulingTest.java similarity index 95% rename from rqueue-core/src/test/java/com/github/sonus21/rqueue/core/MessageSchedulingTest.java rename to rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/core/MessageSchedulingTest.java index 9601049bb..ddede4a7b 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/MessageSchedulingTest.java +++ b/rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/core/MessageSchedulingTest.java @@ -14,7 +14,7 @@ * */ -package com.github.sonus21.rqueue.core; +package com.github.sonus21.rqueue.redis.core; import static com.github.sonus21.rqueue.utils.TimeoutUtils.sleep; import static com.github.sonus21.rqueue.utils.TimeoutUtils.waitFor; @@ -27,9 +27,11 @@ import com.github.sonus21.rqueue.CoreUnitTest; import com.github.sonus21.rqueue.config.RqueueConfig; import com.github.sonus21.rqueue.config.RqueueSchedulerConfig; -import com.github.sonus21.rqueue.core.ProcessingQueueMessageSchedulerTest.ProcessingQTestMessageScheduler; +import com.github.sonus21.rqueue.core.EndpointRegistry; +import com.github.sonus21.rqueue.core.RqueueRedisListenerContainerFactory; import com.github.sonus21.rqueue.listener.QueueDetail; import com.github.sonus21.rqueue.models.event.RqueueBootstrapEvent; +import com.github.sonus21.rqueue.redis.core.ProcessingQueueMessageSchedulerTest.ProcessingQTestMessageScheduler; import com.github.sonus21.rqueue.utils.TestUtils; import com.github.sonus21.rqueue.utils.ThreadUtils; import com.github.sonus21.test.TestTaskScheduler; diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/ProcessingQueueMessageSchedulerTest.java b/rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/core/ProcessingQueueMessageSchedulerTest.java similarity index 97% rename from rqueue-core/src/test/java/com/github/sonus21/rqueue/core/ProcessingQueueMessageSchedulerTest.java rename to rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/core/ProcessingQueueMessageSchedulerTest.java index 24270eb7f..fe9a9ed68 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/ProcessingQueueMessageSchedulerTest.java +++ b/rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/core/ProcessingQueueMessageSchedulerTest.java @@ -14,7 +14,7 @@ * */ -package com.github.sonus21.rqueue.core; +package com.github.sonus21.rqueue.redis.core; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -24,6 +24,7 @@ import com.github.sonus21.TestBase; import com.github.sonus21.rqueue.CoreUnitTest; import com.github.sonus21.rqueue.config.RqueueSchedulerConfig; +import com.github.sonus21.rqueue.core.EndpointRegistry; import com.github.sonus21.rqueue.listener.QueueDetail; import com.github.sonus21.rqueue.utils.TestUtils; import java.util.concurrent.Future; diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/RedisAndNormalSchedulingTest.java b/rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/core/RedisAndNormalSchedulingTest.java similarity index 94% rename from rqueue-core/src/test/java/com/github/sonus21/rqueue/core/RedisAndNormalSchedulingTest.java rename to rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/core/RedisAndNormalSchedulingTest.java index f55970fb7..f58b53010 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/RedisAndNormalSchedulingTest.java +++ b/rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/core/RedisAndNormalSchedulingTest.java @@ -1,4 +1,4 @@ -package com.github.sonus21.rqueue.core; +package com.github.sonus21.rqueue.redis.core; import static com.github.sonus21.rqueue.utils.TimeoutUtils.sleep; import static com.github.sonus21.rqueue.utils.TimeoutUtils.waitFor; @@ -11,9 +11,11 @@ import com.github.sonus21.rqueue.CoreUnitTest; import com.github.sonus21.rqueue.config.RqueueConfig; import com.github.sonus21.rqueue.config.RqueueSchedulerConfig; -import com.github.sonus21.rqueue.core.ScheduledQueueMessageSchedulerTest.TestScheduledQueueMessageScheduler; +import com.github.sonus21.rqueue.core.EndpointRegistry; +import com.github.sonus21.rqueue.core.RqueueRedisListenerContainerFactory; import com.github.sonus21.rqueue.listener.QueueDetail; import com.github.sonus21.rqueue.models.event.RqueueBootstrapEvent; +import com.github.sonus21.rqueue.redis.core.ScheduledQueueMessageSchedulerTest.TestScheduledQueueMessageScheduler; import com.github.sonus21.rqueue.utils.Constants; import com.github.sonus21.rqueue.utils.TestUtils; import com.github.sonus21.rqueue.utils.ThreadUtils; diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/RedisScheduleTriggerHandlerTest.java b/rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/core/RedisScheduleTriggerHandlerTest.java similarity index 97% rename from rqueue-core/src/test/java/com/github/sonus21/rqueue/core/RedisScheduleTriggerHandlerTest.java rename to rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/core/RedisScheduleTriggerHandlerTest.java index 3000c036c..3ef6a21e0 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/RedisScheduleTriggerHandlerTest.java +++ b/rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/core/RedisScheduleTriggerHandlerTest.java @@ -1,4 +1,4 @@ -package com.github.sonus21.rqueue.core; +package com.github.sonus21.rqueue.redis.core; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -10,6 +10,8 @@ import com.github.sonus21.TestBase; import com.github.sonus21.rqueue.CoreUnitTest; import com.github.sonus21.rqueue.config.RqueueSchedulerConfig; +import com.github.sonus21.rqueue.core.EndpointRegistry; +import com.github.sonus21.rqueue.core.RqueueRedisListenerContainerFactory; import com.github.sonus21.rqueue.listener.QueueDetail; import com.github.sonus21.rqueue.utils.TestUtils; import com.github.sonus21.rqueue.utils.TimeoutUtils; diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/ScheduledQueueMessageSchedulerTest.java b/rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/core/ScheduledQueueMessageSchedulerTest.java similarity index 98% rename from rqueue-core/src/test/java/com/github/sonus21/rqueue/core/ScheduledQueueMessageSchedulerTest.java rename to rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/core/ScheduledQueueMessageSchedulerTest.java index fd55d094b..d37224416 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/ScheduledQueueMessageSchedulerTest.java +++ b/rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/core/ScheduledQueueMessageSchedulerTest.java @@ -14,7 +14,7 @@ * */ -package com.github.sonus21.rqueue.core; +package com.github.sonus21.rqueue.redis.core; import static com.github.sonus21.rqueue.utils.TimeoutUtils.sleep; import static com.github.sonus21.rqueue.utils.TimeoutUtils.waitFor; @@ -34,6 +34,8 @@ import com.github.sonus21.rqueue.CoreUnitTest; import com.github.sonus21.rqueue.config.RqueueConfig; import com.github.sonus21.rqueue.config.RqueueSchedulerConfig; +import com.github.sonus21.rqueue.core.EndpointRegistry; +import com.github.sonus21.rqueue.core.RqueueRedisListenerContainerFactory; import com.github.sonus21.rqueue.listener.QueueDetail; import com.github.sonus21.rqueue.models.event.RqueueBootstrapEvent; import com.github.sonus21.rqueue.utils.TestUtils; diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/dao/RqueueQStatsDaoTest.java b/rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/dao/RqueueQStatsDaoTest.java similarity index 95% rename from rqueue-core/src/test/java/com/github/sonus21/rqueue/dao/RqueueQStatsDaoTest.java rename to rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/dao/RqueueQStatsDaoTest.java index 9448269f3..f832b3a45 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/dao/RqueueQStatsDaoTest.java +++ b/rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/dao/RqueueQStatsDaoTest.java @@ -14,7 +14,7 @@ * */ -package com.github.sonus21.rqueue.dao; +package com.github.sonus21.rqueue.redis.dao; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; @@ -26,11 +26,11 @@ import static org.mockito.Mockito.verify; import com.github.sonus21.TestBase; -import com.github.sonus21.rqueue.CoreUnitTest; import com.github.sonus21.rqueue.common.RqueueRedisTemplate; import com.github.sonus21.rqueue.config.RqueueConfig; -import com.github.sonus21.rqueue.dao.impl.RqueueQStatsDaoImpl; +import com.github.sonus21.rqueue.dao.RqueueQStatsDao; import com.github.sonus21.rqueue.models.db.QueueStatistics; +import com.github.sonus21.rqueue.redis.RedisUnitTest; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -40,7 +40,7 @@ import org.mockito.Mock; import org.mockito.MockitoAnnotations; -@CoreUnitTest +@RedisUnitTest @Slf4j class RqueueQStatsDaoTest extends TestBase { diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/dao/RqueueSystemConfigDaoTest.java b/rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/dao/RqueueSystemConfigDaoTest.java similarity index 84% rename from rqueue-core/src/test/java/com/github/sonus21/rqueue/dao/RqueueSystemConfigDaoTest.java rename to rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/dao/RqueueSystemConfigDaoTest.java index dac40e385..2c596c764 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/dao/RqueueSystemConfigDaoTest.java +++ b/rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/dao/RqueueSystemConfigDaoTest.java @@ -14,7 +14,7 @@ * */ -package com.github.sonus21.rqueue.dao; +package com.github.sonus21.rqueue.redis.dao; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; @@ -27,12 +27,12 @@ import static org.mockito.Mockito.verify; import com.github.sonus21.TestBase; -import com.github.sonus21.rqueue.CoreUnitTest; import com.github.sonus21.rqueue.common.RqueueRedisTemplate; import com.github.sonus21.rqueue.config.RqueueConfig; -import com.github.sonus21.rqueue.dao.impl.RqueueSystemConfigDaoImpl; +import com.github.sonus21.rqueue.dao.RqueueSystemConfigDao; import com.github.sonus21.rqueue.models.db.QueueConfig; -import com.github.sonus21.rqueue.utils.TestUtils; +import com.github.sonus21.rqueue.redis.RedisTestUtils; +import com.github.sonus21.rqueue.redis.RedisUnitTest; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -43,12 +43,12 @@ import org.mockito.Mock; import org.mockito.MockitoAnnotations; -@CoreUnitTest +@RedisUnitTest class RqueueSystemConfigDaoTest extends TestBase { private final String queueName = "job"; - private final String configKey = TestUtils.getQueueConfigKey(queueName); - private final QueueConfig queueConfig = TestUtils.createQueueConfig(queueName); + private final String configKey = RedisTestUtils.getQueueConfigKey(queueName); + private final QueueConfig queueConfig = RedisTestUtils.createQueueConfig(queueName); @Mock private RqueueRedisTemplate rqueueRedisTemplate; @@ -67,26 +67,29 @@ public void init() { @Test void getQConfig() { // default return - assertNull(rqueueSystemConfigDao.getQConfig(TestUtils.getQueueConfigKey(queueName), true)); - doReturn(queueConfig).when(rqueueRedisTemplate).get(TestUtils.getQueueConfigKey(queueName)); + assertNull(rqueueSystemConfigDao.getQConfig(RedisTestUtils.getQueueConfigKey(queueName), true)); + doReturn(queueConfig) + .when(rqueueRedisTemplate) + .get(RedisTestUtils.getQueueConfigKey(queueName)); assertEquals( queueConfig, - rqueueSystemConfigDao.getQConfig(TestUtils.getQueueConfigKey(queueName), false)); + rqueueSystemConfigDao.getQConfig(RedisTestUtils.getQueueConfigKey(queueName), false)); assertEquals( queueConfig, - rqueueSystemConfigDao.getQConfig(TestUtils.getQueueConfigKey(queueName), true)); + rqueueSystemConfigDao.getQConfig(RedisTestUtils.getQueueConfigKey(queueName), true)); verify(rqueueRedisTemplate, times(2)).get(any()); assertEquals( queueConfig, - rqueueSystemConfigDao.getQConfig(TestUtils.getQueueConfigKey(queueName), false)); + rqueueSystemConfigDao.getQConfig(RedisTestUtils.getQueueConfigKey(queueName), false)); verify(rqueueRedisTemplate, times(3)).get(any()); } @Test void findAllQConfig() { List keys = Arrays.asList( - TestUtils.getQueueConfigKey(queueName), TestUtils.getQueueConfigKey("notification")); + RedisTestUtils.getQueueConfigKey(queueName), + RedisTestUtils.getQueueConfigKey("notification")); doReturn(Arrays.asList(queueConfig, null)).when(rqueueRedisTemplate).mget(keys); assertEquals( Collections.singletonList(queueConfig), rqueueSystemConfigDao.findAllQConfig(keys)); @@ -94,7 +97,7 @@ void findAllQConfig() { @Test void saveAllQConfig() { - QueueConfig queueConfig2 = TestUtils.createQueueConfig("notification"); + QueueConfig queueConfig2 = RedisTestUtils.createQueueConfig("notification"); doAnswer(invocation -> { Map configMap = new HashMap<>(); configMap.put(queueConfig.getId(), queueConfig); diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/web/service/RqueueMessageMetadataServiceTest.java b/rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/web/service/RqueueMessageMetadataServiceTest.java similarity index 96% rename from rqueue-core/src/test/java/com/github/sonus21/rqueue/web/service/RqueueMessageMetadataServiceTest.java rename to rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/web/service/RqueueMessageMetadataServiceTest.java index 9a534e3a4..3f487c23e 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/web/service/RqueueMessageMetadataServiceTest.java +++ b/rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/web/service/RqueueMessageMetadataServiceTest.java @@ -14,7 +14,7 @@ * */ -package com.github.sonus21.rqueue.web.service; +package com.github.sonus21.rqueue.redis.web.service; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -37,8 +37,9 @@ import com.github.sonus21.rqueue.dao.RqueueStringDao; import com.github.sonus21.rqueue.models.db.MessageMetadata; import com.github.sonus21.rqueue.models.enums.MessageStatus; +import com.github.sonus21.rqueue.redis.web.RqueueMessageMetadataServiceImpl; +import com.github.sonus21.rqueue.service.RqueueMessageMetadataService; import com.github.sonus21.rqueue.utils.Constants; -import com.github.sonus21.rqueue.web.service.impl.RqueueMessageMetadataServiceImpl; import java.time.Duration; import java.util.Arrays; import java.util.Collections; diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/web/service/RqueueUtilityServiceTest.java b/rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/web/service/RqueueUtilityServiceTest.java similarity index 97% rename from rqueue-core/src/test/java/com/github/sonus21/rqueue/web/service/RqueueUtilityServiceTest.java rename to rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/web/service/RqueueUtilityServiceTest.java index 9342ba404..833c22154 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/web/service/RqueueUtilityServiceTest.java +++ b/rqueue-redis/src/test/java/com/github/sonus21/rqueue/redis/web/service/RqueueUtilityServiceTest.java @@ -14,7 +14,7 @@ * */ -package com.github.sonus21.rqueue.web.service; +package com.github.sonus21.rqueue.redis.web.service; import static com.github.sonus21.rqueue.utils.TestUtils.createQueueConfig; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -46,7 +46,9 @@ import com.github.sonus21.rqueue.models.response.BooleanResponse; import com.github.sonus21.rqueue.models.response.MessageMoveResponse; import com.github.sonus21.rqueue.models.response.StringResponse; -import com.github.sonus21.rqueue.web.service.impl.RqueueUtilityServiceImpl; +import com.github.sonus21.rqueue.redis.web.RqueueUtilityServiceImpl; +import com.github.sonus21.rqueue.service.RqueueMessageMetadataService; +import com.github.sonus21.rqueue.service.RqueueUtilityService; import java.io.Serializable; import java.time.Duration; import java.util.Collections; diff --git a/rqueue-spring-boot-example/src/main/java/com/github/sonus21/rqueue/example/BaseListener.java b/rqueue-spring-boot-example/src/main/java/com/github/sonus21/rqueue/example/BaseListener.java new file mode 100644 index 000000000..764b41b0a --- /dev/null +++ b/rqueue-spring-boot-example/src/main/java/com/github/sonus21/rqueue/example/BaseListener.java @@ -0,0 +1,45 @@ +package com.github.sonus21.rqueue.example; + +import com.github.sonus21.rqueue.core.RqueueMessageManager; +import com.github.sonus21.rqueue.utils.TimeoutUtils; +import java.util.Random; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; + +@Slf4j +public abstract class BaseListener { + + protected static final Random random = new Random(); + + @Autowired + @Lazy + protected RqueueMessageManager rqueueMessageManager; + + @Value("${job.fail.percentage:0}") + protected int percentageFailure; + + @Value("${job.execution.interval:100}") + protected int jobExecutionTime; + + protected int count; + + protected boolean shouldFail() { + if (percentageFailure == 0) { + return false; + } + if (percentageFailure >= 100) { + return true; + } + return random.nextInt(100) < percentageFailure; + } + + protected void execute(String msg, Object any, boolean failingEnabled) { + log.info(msg, any); + TimeoutUtils.sleep(random.nextInt(jobExecutionTime)); + if (failingEnabled && shouldFail()) { + throw new IllegalArgumentException("Failing On Purpose " + any); + } + } +} diff --git a/rqueue-spring-boot-example/src/main/java/com/github/sonus21/rqueue/example/JobListener.java b/rqueue-spring-boot-example/src/main/java/com/github/sonus21/rqueue/example/JobListener.java new file mode 100644 index 000000000..d569b61e7 --- /dev/null +++ b/rqueue-spring-boot-example/src/main/java/com/github/sonus21/rqueue/example/JobListener.java @@ -0,0 +1,25 @@ +package com.github.sonus21.rqueue.example; + +import com.github.sonus21.rqueue.annotation.RqueueHandler; +import com.github.sonus21.rqueue.annotation.RqueueListener; +import org.springframework.stereotype.Component; + +@RqueueListener( + value = "job-queue", + deadLetterQueue = "job-morgue", + numRetries = "2", + deadLetterQueueListenerEnabled = "false", + concurrency = "10-20") +@Component +public class JobListener extends BaseListener { + + @RqueueHandler + public void onJobMessage(Job job) { + execute("job-queue: {}", job, true); + } + + @RqueueHandler(primary = true) + public void onLinkedInJobMessage(Job job) { + execute("linkedin:job-queue: {}", job, true); + } +} diff --git a/rqueue-spring-boot-example/src/main/java/com/github/sonus21/rqueue/example/MessageListener.java b/rqueue-spring-boot-example/src/main/java/com/github/sonus21/rqueue/example/MessageListener.java index 2f010442e..9766af269 100644 --- a/rqueue-spring-boot-example/src/main/java/com/github/sonus21/rqueue/example/MessageListener.java +++ b/rqueue-spring-boot-example/src/main/java/com/github/sonus21/rqueue/example/MessageListener.java @@ -17,50 +17,14 @@ package com.github.sonus21.rqueue.example; import com.github.sonus21.rqueue.annotation.RqueueListener; -import com.github.sonus21.rqueue.core.RqueueMessageManager; import com.github.sonus21.rqueue.listener.RqueueMessageHeaders; -import com.github.sonus21.rqueue.utils.TimeoutUtils; -import java.util.Random; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.messaging.handler.annotation.Header; import org.springframework.stereotype.Component; @Component @Slf4j -public class MessageListener { - - private static final Random random = new Random(); - - @Autowired - private RqueueMessageManager rqueueMessageManager; - - @Value("${job.fail.percentage:0}") - private int percentageFailure; - - @Value("${job.execution.interval:100}") - private int jobExecutionTime; - - private int count; - - protected boolean shouldFail() { - if (percentageFailure == 0) { - return false; - } - if (percentageFailure >= 100) { - return true; - } - return random.nextInt(100) < percentageFailure; - } - - protected void execute(String msg, Object any, boolean failingEnabled) { - log.info(msg, any); - TimeoutUtils.sleep(random.nextInt(jobExecutionTime)); - if (failingEnabled && shouldFail()) { - throw new IllegalArgumentException("Failing On Purpose " + any); - } - } +public class MessageListener extends BaseListener { @RqueueListener(value = "${rqueue.simple.queue}") public void onSimpleMessage(String message) { @@ -75,16 +39,6 @@ public void onMessage(String message) { execute("delay: {}", message, true); } - @RqueueListener( - value = "job-queue", - deadLetterQueue = "job-morgue", - numRetries = "2", - deadLetterQueueListenerEnabled = "false", - concurrency = "10-20") - public void onJobMessage(Job job) { - execute("job-queue: {}", job, true); - } - @RqueueListener( value = "sch-job-queue", deadLetterQueue = "job-morgue", diff --git a/rqueue-spring-boot-nats-example/README.md b/rqueue-spring-boot-nats-example/README.md new file mode 100644 index 000000000..2d671f4c9 --- /dev/null +++ b/rqueue-spring-boot-nats-example/README.md @@ -0,0 +1,77 @@ +# rqueue-spring-boot-nats-example + +Spring Boot example app using **NATS / JetStream** as the Rqueue backend, mirroring +[`rqueue-spring-boot-example`](../rqueue-spring-boot-example) (which uses Redis). + +The application code is identical to the redis example modulo two small things: + +- **Delayed and periodic enqueue endpoints are removed.** The v1 NATS broker doesn't model + delayed or scheduled delivery — the redis backend's ZSET schedulers don't exist on the NATS + side, and the broker throws `UnsupportedOperationException` for those calls. +- **`application.properties` selects the NATS backend** via `rqueue.backend=nats` and points + `rqueue.nats.connection.url` at a JetStream-enabled `nats-server`. + +Backend selection is a property switch only — the listener / controller / domain code is +unchanged from the redis example, which is the whole point of the pluggable-backend split. + +## Running locally + +Start a JetStream-enabled NATS server (any one of these works): + +```sh +# native binary +nats-server -js + +# docker +docker run -p 4222:4222 nats:latest -js +``` + +Then: + +```sh +./gradlew :rqueue-spring-boot-nats-example:bootRun +``` + +Once the app is up: + +```sh +# enqueue a String to a queue (default queue is "simple-queue", from application.properties) +curl 'http://localhost:8080/push?q=simple-queue&msg=hello' + +# enqueue a Job object to job-queue (with DLQ wired to job-morgue) +curl http://localhost:8080/job +``` + +Watch the logs for `simple: hello` / `job-queue: Job(id=…)` from `MessageListener`. + +## Inspecting the JetStream state + +`nats stream ls` will show: + +``` +rqueue-js-simple-queue +rqueue-js-job-queue +rqueue-js-job-queue-dlq +rqueue-js-job-morgue +``` + +(The `rqueue-js-` prefix is the default; configure via `rqueue.nats.naming.streamPrefix`.) + +`nats kv ls` will show the six shared KV buckets used by the NATS-backed daos +(`rqueue-jobs`, `rqueue-locks`, `rqueue-message-metadata`, etc.). See the README's +"NATS backend" section in the repo root for the full table. + +## Locked-down JetStream accounts + +If your NATS account can't run `add_stream` / `kv_create` at runtime, set: + +```properties +rqueue.nats.auto-create-streams=false +rqueue.nats.auto-create-consumers=false +rqueue.nats.auto-create-dlq-stream=false +rqueue.nats.auto-create-kv-buckets=false +``` + +…and pre-create the streams + buckets per the root README. `NatsStreamValidator` / +`NatsKvBucketValidator` will fail boot deterministically with the list of missing +streams / buckets if any are not present. diff --git a/rqueue-spring-boot-nats-example/build.gradle b/rqueue-spring-boot-nats-example/build.gradle new file mode 100644 index 000000000..90baa6973 --- /dev/null +++ b/rqueue-spring-boot-nats-example/build.gradle @@ -0,0 +1,20 @@ +plugins { + id "org.springframework.boot" version "${springBootVersion}" + id "war" +} +dependencies { + implementation project(":rqueue-spring-boot-starter") + // rqueue-nats brings the JetStream broker, KV-backed daos, NatsStreamValidator, etc. + // It's a runtime-conditional dependency on the starter side (gated by rqueue.backend=nats), + // so the example pins it explicitly. + implementation project(":rqueue-nats") + implementation "org.springframework.boot:spring-boot-starter-web:${springBootVersion}" + // jnats is api-exposed by rqueue-nats, but pinning it here keeps the build readable. + implementation "io.nats:jnats:${natsVersion}" + // https://mvnrepository.com/artifact/ch.qos.logback/logback-core + implementation "ch.qos.logback:logback-core:${logbackVersion}" + // https://mvnrepository.com/artifact/ch.qos.logback/logback-classic + implementation "ch.qos.logback:logback-classic:${logbackVersion}" + annotationProcessor "org.springframework.boot:spring-boot-configuration-processor:${springBootVersion}" + providedRuntime "org.springframework.boot:spring-boot-starter-tomcat:${springBootVersion}" +} diff --git a/rqueue-spring-boot-nats-example/src/main/java/com/github/sonus21/rqueue/example/Controller.java b/rqueue-spring-boot-nats-example/src/main/java/com/github/sonus21/rqueue/example/Controller.java new file mode 100644 index 000000000..55e9c2527 --- /dev/null +++ b/rqueue-spring-boot-nats-example/src/main/java/com/github/sonus21/rqueue/example/Controller.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ + +package com.github.sonus21.rqueue.example; + +import com.github.sonus21.rqueue.core.RqueueMessageEnqueuer; +import com.github.sonus21.rqueue.utils.StringUtils; +import java.util.UUID; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +/** + * Same shape as the redis example's controller, with delayed / periodic endpoints removed — + * the v1 NATS broker doesn't support delayed delivery or periodic enqueue and would throw + * {@code UnsupportedOperationException} at runtime if those code paths were hit. + */ +@RestController +@AllArgsConstructor(onConstructor = @__(@Autowired)) +@Slf4j +public class Controller { + + private final RqueueMessageEnqueuer rqueueMessageEnqueuer; + + @GetMapping(value = "/push") + public String push(String q, String msg) { + String messageId = rqueueMessageEnqueuer.enqueue(q, msg); + log.info("Message {}", msg); + return "Message sent successfully, message id " + messageId; + } + + private String getQueue(String queue) { + if (queue == null) { + return "job-queue"; + } + return queue; + } + + private Job getJob(String message) { + Job job = new Job(); + job.setId(UUID.randomUUID().toString()); + if (!StringUtils.isEmpty(message)) { + job.setMessage(message); + } else { + job.setMessage("Hi this is " + job.getId()); + } + return job; + } + + @GetMapping("job") + public String sendJobNotification( + @RequestParam(required = false) String msg, @RequestParam(required = false) String q) { + Job job = getJob(msg); + String messageId = rqueueMessageEnqueuer.enqueue(getQueue(q), job); + job.setMessage(messageId); + return job.toString(); + } +} diff --git a/rqueue-spring-boot-nats-example/src/main/java/com/github/sonus21/rqueue/example/Job.java b/rqueue-spring-boot-nats-example/src/main/java/com/github/sonus21/rqueue/example/Job.java new file mode 100644 index 000000000..1a75a8ed6 --- /dev/null +++ b/rqueue-spring-boot-nats-example/src/main/java/com/github/sonus21/rqueue/example/Job.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ + +package com.github.sonus21.rqueue.example; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +@Getter +@Setter +@ToString +public class Job { + + private String id; + private String message; + private String messageId; +} diff --git a/rqueue-spring-boot-nats-example/src/main/java/com/github/sonus21/rqueue/example/MessageListener.java b/rqueue-spring-boot-nats-example/src/main/java/com/github/sonus21/rqueue/example/MessageListener.java new file mode 100644 index 000000000..38b5cd7c2 --- /dev/null +++ b/rqueue-spring-boot-nats-example/src/main/java/com/github/sonus21/rqueue/example/MessageListener.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ + +package com.github.sonus21.rqueue.example; + +import com.github.sonus21.rqueue.annotation.RqueueListener; +import com.github.sonus21.rqueue.utils.TimeoutUtils; +import java.util.Random; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +/** + * Same listener shape as the redis example, minus the delayed-queue and scheduled-job listeners + * (those rely on Redis-only ZSET-backed schedulers; the v1 NATS broker delegates redelivery to + * JetStream's own {@code AckWait} timer instead). + */ +@Component +@Slf4j +public class MessageListener { + + private static final Random random = new Random(); + + @Value("${job.fail.percentage:0}") + private int percentageFailure; + + @Value("${job.execution.interval:100}") + private int jobExecutionTime; + + protected boolean shouldFail() { + if (percentageFailure == 0) { + return false; + } + if (percentageFailure >= 100) { + return true; + } + return random.nextInt(100) < percentageFailure; + } + + protected void execute(String msg, Object any, boolean failingEnabled) { + log.info(msg, any); + TimeoutUtils.sleep(random.nextInt(jobExecutionTime)); + if (failingEnabled && shouldFail()) { + throw new IllegalArgumentException("Failing On Purpose " + any); + } + } + + @RqueueListener(value = "${rqueue.simple.queue}") + public void onSimpleMessage(String message) { + execute("simple: {}", message, false); + } + + @RqueueListener( + value = "job-queue", + deadLetterQueue = "job-queue-linkedin-dlq", + numRetries = "2", + concurrency = "10-20", + consumerName = "linkedin-search") + public void onJobMessage(Job job) { + execute("job-queue-linkedin: {}", job, true); + } + + @RqueueListener( + value = "job-queue", + numRetries = "2", + deadLetterQueue = "job-queue-google-dlq", + concurrency = "10-20", + consumerName = "google-search") + public void onJobMessageGooglSearch(Job job) { + execute("job-queue-google: {}", job, true); + } + + @RqueueListener(value = "job-morgue", numRetries = "1", concurrency = "1-3") + public void onJobDlqMessage(Job job) { + execute("job-morgue: {}", job, true); + } +} diff --git a/rqueue-spring-boot-nats-example/src/main/java/com/github/sonus21/rqueue/example/MvcConfig.java b/rqueue-spring-boot-nats-example/src/main/java/com/github/sonus21/rqueue/example/MvcConfig.java new file mode 100644 index 000000000..1febc94bf --- /dev/null +++ b/rqueue-spring-boot-nats-example/src/main/java/com/github/sonus21/rqueue/example/MvcConfig.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ + +package com.github.sonus21.rqueue.example; + +import com.github.sonus21.rqueue.utils.StringUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.servlet.config.annotation.EnableWebMvc; +import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; + +@EnableWebMvc +@Configuration +public class MvcConfig implements WebMvcConfigurer { + + @Value("${rqueue.web.url.prefix:}") + private String rqueueWebUrlPrefix; + + @Override + public void addResourceHandlers(ResourceHandlerRegistry registry) { + if (!registry.hasMappingForPattern("/webjars/**")) { + registry + .addResourceHandler("/webjars/**") + .addResourceLocations("classpath:/META-INF/resources/webjars/"); + } + if (!StringUtils.isEmpty(rqueueWebUrlPrefix)) { + registry + .addResourceHandler(rqueueWebUrlPrefix + "/**") + .addResourceLocations("classpath:/public/"); + } + if (!registry.hasMappingForPattern("/**")) { + registry.addResourceHandler("/**").addResourceLocations("classpath:/public/"); + } + } +} diff --git a/rqueue-spring-boot-nats-example/src/main/java/com/github/sonus21/rqueue/example/RQueueNatApplication.java b/rqueue-spring-boot-nats-example/src/main/java/com/github/sonus21/rqueue/example/RQueueNatApplication.java new file mode 100644 index 000000000..04b0ec68b --- /dev/null +++ b/rqueue-spring-boot-nats-example/src/main/java/com/github/sonus21/rqueue/example/RQueueNatApplication.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ + +package com.github.sonus21.rqueue.example; + +import com.github.sonus21.rqueue.config.SimpleRqueueListenerContainerFactory; +import com.github.sonus21.rqueue.utils.Constants; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.Bean; + +/** + * Mirror of the redis-backed {@code rqueue-spring-boot-example} for the NATS / JetStream backend. + * Selecting the backend is a property switch only (see {@code application.properties}); the + * application code is unchanged from the redis example, which is the whole point of the + * pluggable-backend split. + */ +@SpringBootApplication +public class RQueueNatApplication { + + @Value("${workers.count:3}") + private int workersCount; + + public static void main(String[] args) { + SpringApplication.run(RQueueNatApplication.class, args); + } + + @Bean + public SimpleRqueueListenerContainerFactory simpleRqueueListenerContainerFactory() { + SimpleRqueueListenerContainerFactory simpleRqueueListenerContainerFactory = + new SimpleRqueueListenerContainerFactory(); + simpleRqueueListenerContainerFactory.setMaxNumWorkers(workersCount); + simpleRqueueListenerContainerFactory.setPollingInterval(Constants.ONE_MILLI); + return simpleRqueueListenerContainerFactory; + } +} diff --git a/rqueue-spring-boot-nats-example/src/main/resources/application.properties b/rqueue-spring-boot-nats-example/src/main/resources/application.properties new file mode 100644 index 000000000..0cc4bf0a1 --- /dev/null +++ b/rqueue-spring-boot-nats-example/src/main/resources/application.properties @@ -0,0 +1,55 @@ +# +# Copyright (c) 2026 Sonu Kumar +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and limitations under the License. +# +# + +# Backend selection — flip the existing redis-backed example onto NATS by +# setting backend=nats and pointing rqueue.nats.connection.url at a JetStream- +# enabled server. Everything else (queue names, listeners, controller) is +# identical to the redis example, which is the point of the pluggable split. +rqueue.backend=nats +server.port=9090 + +# NATS / JetStream connection. Default is the public-binding nats-server URL +# from a vanilla `nats-server -js` install. For docker, point at the running +# container instead (e.g. nats://nats:4222). +rqueue.nats.connection.url=nats://localhost:4222 + +# Provisioning flags. Defaults are true (auto-create on first use); flip any +# to false in restricted JetStream accounts and pre-create the streams / +# buckets per the README. +rqueue.nats.auto-create-streams=true +rqueue.nats.auto-create-consumers=true +rqueue.nats.auto-create-dlq-stream=true +rqueue.nats.auto-create-kv-buckets=true + +# Queue names. Same defaults as the redis example minus the two delayed +# queues — v1 NATS doesn't model delayed delivery, so a delay-queue listener +# wouldn't have anything to redeliver from. +rqueue.simple.queue=simple-queue + +job.fail.percentage=30 +job.execution.interval=2000 + +# Standard actuator + Prometheus export wiring; identical to the redis example. +rqueue.metrics.tags.suite=JMeter +rqueue.metrics.count.failure=true +rqueue.metrics.count.execution=true +management.prometheus.metrics.export.enabled=true +management.endpoints.web.exposure.include=* +management.endpoint.beans.enabled=true +management.endpoint.health.enabled=true + +workers.count=100 +scale.enabled=true diff --git a/rqueue-spring-boot-nats-example/src/main/resources/logback.xml b/rqueue-spring-boot-nats-example/src/main/resources/logback.xml new file mode 100644 index 000000000..d6a612743 --- /dev/null +++ b/rqueue-spring-boot-nats-example/src/main/resources/logback.xml @@ -0,0 +1,44 @@ + + + + + + [%date{dd-MM-yyyy HH:mm:ss.SSS}] [%thread] %-5level %X{traceId:-} %X{spanId:-} ${PID:-} %logger{36} - %msg%n + + + + + [%date{dd-MM-yyyy HH:mm:ss.SSS}] [%thread] %-5level %X{traceId:-} %X{spanId:-} ${PID:-} %logger{36} - %msg%n + + log/app.log + + log/app.%d{yyyy-MM-dd-HH}.log + 30 + 200MB + + + + + + + + + + + + + \ No newline at end of file diff --git a/rqueue-spring-boot-starter/build.gradle b/rqueue-spring-boot-starter/build.gradle index 828d52436..ac5dc7468 100644 --- a/rqueue-spring-boot-starter/build.gradle +++ b/rqueue-spring-boot-starter/build.gradle @@ -42,13 +42,32 @@ mavenPublishing { dependencies { api project(":rqueue-core") + // Default-on dashboard module. Consumers who want a headless worker can rqueue-web. + api project(":rqueue-web") + api project(":rqueue-redis") api "org.springframework.boot:spring-boot-starter-data-redis:${springBootVersion}" api "org.springframework.boot:spring-boot-starter-actuator:${springBootVersion}" + + // NATS support is opt-in. Auto-config classes reference rqueue-nats types but are gated by + // @ConditionalOnClass(io.nats.client.JetStream.class) and @ConditionalOnProperty + // (rqueue.backend=nats). Users who want NATS pull rqueue-nats explicitly; the starter alone + // keeps its existing Redis-only behavior with zero new transitive deps. + compileOnly project(":rqueue-nats") + compileOnly "io.nats:jnats:${natsVersion}" + + // Generates spring-configuration-metadata.json from @ConfigurationProperties classes + // (RqueueNatsProperties primarily) so IDE autocomplete and the Spring Boot metadata viewer + // surface every rqueue.nats.* key. Processor only — no runtime dep added. + annotationProcessor "org.springframework.boot:spring-boot-configuration-processor:${springBootVersion}" + testImplementation project(":rqueue-spring-common-test") + testImplementation project(":rqueue-nats") testImplementation "org.springframework.boot:spring-boot-starter-test:${springBootVersion}" testImplementation "org.springframework.boot:spring-boot-starter-webflux:${springBootVersion}" testImplementation "org.springframework.boot:spring-boot-webtestclient:${springBootVersion}" testImplementation "org.springframework.boot:spring-boot-starter-data-redis-reactive:${springBootVersion}" testImplementation "org.springframework.boot:spring-boot-starter-data-jpa:${springBootVersion}" testImplementation "org.springframework.boot:spring-boot-devtools:${springBootVersion}" + testImplementation "org.testcontainers:testcontainers:${testcontainersVersion}" + testImplementation "org.testcontainers:junit-jupiter:${testcontainersVersion}" } diff --git a/rqueue-spring-boot-starter/src/main/java/com/github/sonus21/rqueue/spring/boot/RqueueListenerAutoConfig.java b/rqueue-spring-boot-starter/src/main/java/com/github/sonus21/rqueue/spring/boot/RqueueListenerAutoConfig.java index 9e0fdd7d8..9ac5c4fcd 100644 --- a/rqueue-spring-boot-starter/src/main/java/com/github/sonus21/rqueue/spring/boot/RqueueListenerAutoConfig.java +++ b/rqueue-spring-boot-starter/src/main/java/com/github/sonus21/rqueue/spring/boot/RqueueListenerAutoConfig.java @@ -28,6 +28,7 @@ import com.github.sonus21.rqueue.core.impl.RqueueEndpointManagerImpl; import com.github.sonus21.rqueue.core.impl.RqueueMessageEnqueuerImpl; import com.github.sonus21.rqueue.core.impl.RqueueMessageManagerImpl; +import com.github.sonus21.rqueue.core.spi.MessageBroker; import com.github.sonus21.rqueue.listener.RqueueMessageHandler; import com.github.sonus21.rqueue.listener.RqueueMessageListenerContainer; import com.github.sonus21.rqueue.utils.condition.ReactiveEnabled; @@ -40,33 +41,44 @@ import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.DependsOn; +import org.springframework.context.annotation.Import; @Configuration @AutoConfigureAfter(DataRedisAutoConfiguration.class) -@ComponentScan({"com.github.sonus21.rqueue.web", "com.github.sonus21.rqueue.dao"}) +@ComponentScan({ + "com.github.sonus21.rqueue.web", + "com.github.sonus21.rqueue.dao", + "com.github.sonus21.rqueue.nats", +}) @Conditional({RqueueEnabled.class}) +@Import(RqueueRedisConfigImportSelector.class) public class RqueueListenerAutoConfig extends RqueueListenerBaseConfig { @Bean @ConditionalOnMissingBean - public RqueueMessageHandler rqueueMessageHandler() { - return simpleRqueueListenerContainerFactory.getRqueueMessageHandler( - getMessageConverterProvider()); + public RqueueMessageHandler rqueueMessageHandler(MessageBroker messageBroker) { + RqueueMessageHandler handler = + simpleRqueueListenerContainerFactory.getRqueueMessageHandler(getMessageConverterProvider()); + handler.setPrimaryHandlerDispatchEnabled( + messageBroker.capabilities().usesPrimaryHandlerDispatch()); + return handler; } @Bean @DependsOn("rqueueConfig") @ConditionalOnMissingBean public RqueueMessageListenerContainer rqueueMessageListenerContainer( - RqueueMessageHandler rqueueMessageHandler) { + RqueueMessageHandler rqueueMessageHandler, MessageBroker messageBroker) { simpleRqueueListenerContainerFactory.setRqueueMessageHandler(rqueueMessageHandler); + if (simpleRqueueListenerContainerFactory.getMessageBroker() == null) { + simpleRqueueListenerContainerFactory.setMessageBroker(messageBroker); + } return simpleRqueueListenerContainerFactory.createMessageListenerContainer(); } @Bean @ConditionalOnMissingBean - public RqueueMessageTemplate rqueueMessageTemplate( - RqueueConfig rqueueConfig, RqueueMessageHandler rqueueMessageHandler) { + public RqueueMessageTemplate rqueueMessageTemplate(RqueueConfig rqueueConfig) { return getMessageTemplate(rqueueConfig); } @@ -75,9 +87,11 @@ public RqueueMessageTemplate rqueueMessageTemplate( public RqueueMessageManager rqueueMessageManager( RqueueMessageHandler rqueueMessageHandler, RqueueMessageTemplate rqueueMessageTemplate, + MessageBroker messageBroker, RqueueMessageIdGenerator rqueueMessageIdGenerator) { return new RqueueMessageManagerImpl( rqueueMessageTemplate, + messageBroker, rqueueMessageHandler.getMessageConverter(), simpleRqueueListenerContainerFactory.getMessageHeaders(), rqueueMessageIdGenerator); @@ -88,9 +102,11 @@ public RqueueMessageManager rqueueMessageManager( public RqueueEndpointManager rqueueEndpointManager( RqueueMessageHandler rqueueMessageHandler, RqueueMessageTemplate rqueueMessageTemplate, + MessageBroker messageBroker, RqueueMessageIdGenerator rqueueMessageIdGenerator) { return new RqueueEndpointManagerImpl( rqueueMessageTemplate, + messageBroker, rqueueMessageHandler.getMessageConverter(), simpleRqueueListenerContainerFactory.getMessageHeaders(), rqueueMessageIdGenerator); @@ -101,9 +117,11 @@ public RqueueEndpointManager rqueueEndpointManager( public RqueueMessageEnqueuer rqueueMessageEnqueuer( RqueueMessageHandler rqueueMessageHandler, RqueueMessageTemplate rqueueMessageTemplate, + MessageBroker messageBroker, RqueueMessageIdGenerator rqueueMessageIdGenerator) { return new RqueueMessageEnqueuerImpl( rqueueMessageTemplate, + messageBroker, rqueueMessageHandler.getMessageConverter(), simpleRqueueListenerContainerFactory.getMessageHeaders(), rqueueMessageIdGenerator); @@ -115,9 +133,11 @@ public RqueueMessageEnqueuer rqueueMessageEnqueuer( public ReactiveRqueueMessageEnqueuer reactiveRqueueMessageEnqueuer( RqueueMessageHandler rqueueMessageHandler, RqueueMessageTemplate rqueueMessageTemplate, + MessageBroker messageBroker, RqueueMessageIdGenerator rqueueMessageIdGenerator) { return new ReactiveRqueueMessageEnqueuerImpl( rqueueMessageTemplate, + messageBroker, rqueueMessageHandler.getMessageConverter(), simpleRqueueListenerContainerFactory.getMessageHeaders(), rqueueMessageIdGenerator); diff --git a/rqueue-spring-boot-starter/src/main/java/com/github/sonus21/rqueue/spring/boot/RqueueNatsAutoConfig.java b/rqueue-spring-boot-starter/src/main/java/com/github/sonus21/rqueue/spring/boot/RqueueNatsAutoConfig.java new file mode 100644 index 000000000..d0a5237be --- /dev/null +++ b/rqueue-spring-boot-starter/src/main/java/com/github/sonus21/rqueue/spring/boot/RqueueNatsAutoConfig.java @@ -0,0 +1,245 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ +package com.github.sonus21.rqueue.spring.boot; + +import com.github.sonus21.rqueue.config.RqueueConfig; +import com.github.sonus21.rqueue.core.spi.MessageBroker; +import com.github.sonus21.rqueue.metrics.RqueueQueueMetricsProvider; +import com.github.sonus21.rqueue.nats.RqueueNatsConfig; +import com.github.sonus21.rqueue.nats.internal.NatsProvisioner; +import com.github.sonus21.rqueue.nats.js.JetStreamMessageBroker; +import com.github.sonus21.rqueue.nats.js.NatsStreamValidator; +import com.github.sonus21.rqueue.nats.kv.NatsKvBucketValidator; +import com.github.sonus21.rqueue.nats.metrics.NatsRqueueQueueMetricsProvider; +import com.github.sonus21.rqueue.serdes.RqJacksonSerDes; +import com.github.sonus21.rqueue.serdes.SerializationUtils; +import com.github.sonus21.rqueue.utils.StringUtils; +import io.nats.client.Connection; +import io.nats.client.JetStream; +import io.nats.client.JetStreamManagement; +import io.nats.client.Nats; +import io.nats.client.Options; +import java.io.IOException; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.boot.autoconfigure.AutoConfigureBefore; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.DependsOn; + +/** + * Auto-configuration that wires a JetStream-backed {@link MessageBroker} when + * {@code rqueue.backend=nats} and the jnats client is on the classpath. + * + *

This auto-config runs before {@link RqueueListenerAutoConfig} so that the broker bean is + * available for the listener container factory to consume; the existing Redis broker bean uses + * {@code @ConditionalOnMissingBean(MessageBroker.class)} so it backs off when this one is present. + */ +@AutoConfiguration +@AutoConfigureBefore(RqueueListenerAutoConfig.class) +@ConditionalOnClass(JetStream.class) +@ConditionalOnProperty(name = "rqueue.backend", havingValue = "nats") +@EnableConfigurationProperties(RqueueNatsProperties.class) +public class RqueueNatsAutoConfig { + + @Bean + @ConditionalOnMissingBean + public Connection natsConnection(RqueueNatsProperties props) throws IOException { + Options.Builder ob = new Options.Builder(); + RqueueNatsProperties.Connection c = props.getConnection(); + if (!StringUtils.isEmpty(c.getUrl())) { + ob.server(c.getUrl()); + } else { + ob.server(Options.DEFAULT_URL); + } + if (!StringUtils.isEmpty(c.getToken())) { + ob.token(c.getToken().toCharArray()); + } else if (!StringUtils.isEmpty(c.getUsername()) && !StringUtils.isEmpty(c.getPassword())) { + ob.userInfo(c.getUsername(), c.getPassword()); + } + if (!StringUtils.isEmpty(c.getConnectionName())) { + ob.connectionName(c.getConnectionName()); + } + if (c.getConnectTimeout() != null) { + ob.connectionTimeout(c.getConnectTimeout()); + } + if (c.getReconnectWait() != null) { + ob.reconnectWait(c.getReconnectWait()); + } + if (c.getMaxReconnects() >= 0) { + ob.maxReconnects(c.getMaxReconnects()); + } + if (c.getPingInterval() != null) { + ob.pingInterval(c.getPingInterval()); + } + Connection connection; + try { + connection = Nats.connect(ob.build()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted while connecting to NATS", e); + } + // Validate KV buckets here — inside the Connection bean factory — so every other NATS + // bean (broker, daos, registry, lock manager) that injects Connection is guaranteed to + // see a validated cluster. Spring's @Order/@Priority don't control bean creation order; + // anchoring the check on the Connection itself does. + NatsKvBucketValidator.validate(connection, props.isAutoCreateKvBuckets()); + return connection; + } + + @Bean + @ConditionalOnMissingBean + public JetStream jetStream(Connection connection) throws IOException { + return connection.jetStream(); + } + + @Bean + @ConditionalOnMissingBean + public JetStreamManagement jetStreamManagement(Connection connection) throws IOException { + return connection.jetStreamManagement(); + } + + @Bean + @ConditionalOnMissingBean(MessageBroker.class) + public MessageBroker jetStreamMessageBroker( + Connection connection, + JetStream jetStream, + JetStreamManagement jetStreamManagement, + NatsProvisioner natsProvisioner, + RqueueNatsProperties props) { + return JetStreamMessageBroker.builder() + .connection(connection) + .jetStream(jetStream) + .management(jetStreamManagement) + .config(toBrokerConfig(props)) + .provisioner(natsProvisioner) + .build(); + } + + @Bean + @ConditionalOnMissingBean(RqueueQueueMetricsProvider.class) + public RqueueQueueMetricsProvider natsRqueueQueueMetricsProvider( + JetStreamManagement jetStreamManagement, RqueueNatsProperties props) { + return new NatsRqueueQueueMetricsProvider(jetStreamManagement, toBrokerConfig(props)); + } + + /** + * Boot-time stream / DLQ existence guard. Implements {@code SmartInitializingSingleton} so it + * runs after every {@code @RqueueListener} has registered with {@code EndpointRegistry} but + * before {@code SmartLifecycle.start()} spawns the message pollers — otherwise pollers race the + * validator and surface {@code stream not found [10059]}. Removes the per-publish + * {@code getStreamInfo} round-trip from the broker hot path. + */ + @Bean + @ConditionalOnMissingBean(NatsStreamValidator.class) + public NatsStreamValidator natsStreamValidator( + NatsProvisioner natsProvisioner, + RqueueNatsProperties props, + ObjectProvider rqueueConfigProvider) { + return new NatsStreamValidator( + natsProvisioner, toBrokerConfig(props), rqueueConfigProvider.getIfAvailable()); + } + + /** + * Bean form of the KV-bucket validator. Other NATS beans {@code @DependsOn} this name so it runs + * before they are constructed. The flag is sourced from {@link RqueueNatsProperties} — + * {@code rqueue-nats} itself never reads {@code rqueue.nats.*} keys directly. + */ + @Bean + @ConditionalOnMissingBean(NatsKvBucketValidator.class) + public NatsKvBucketValidator natsKvBucketValidator( + Connection connection, RqueueNatsProperties props) { + return new NatsKvBucketValidator(connection, props.isAutoCreateKvBuckets()); + } + + /** + * Shared {@link com.github.sonus21.rqueue.serdes.RqueueSerDes} for the NATS backend. Backed by + * Jackson with the same configuration as the rest of rqueue + * ({@code FAIL_ON_UNKNOWN_PROPERTIES=false}, auto-detected modules) so values written to KV + * buckets are readable via {@code nats kv get}. + */ + @Bean + @ConditionalOnMissingBean(com.github.sonus21.rqueue.serdes.RqueueSerDes.class) + public com.github.sonus21.rqueue.serdes.RqueueSerDes natsSerDes() { + return new RqJacksonSerDes(SerializationUtils.getObjectMapper()); + } + + @Bean + @ConditionalOnMissingBean(NatsProvisioner.class) + @DependsOn("natsKvBucketValidator") + public NatsProvisioner natsProvisioner( + Connection connection, JetStreamManagement jetStreamManagement, RqueueNatsProperties props) + throws IOException { + return new NatsProvisioner(connection, jetStreamManagement, toBrokerConfig(props)); + } + + /** + * NATS-side {@link com.github.sonus21.rqueue.repository.MessageBrowsingRepository} powering the + * dashboard's data-explorer and queue-detail panels. Maps Redis-style queue names to JetStream + * streams and returns actual message counts from the broker. JetStream KV doesn't model arbitrary + * keyed reads, so throws {@code BackendCapabilityException} (mapped to HTTP 501 by + * {@code RqueueWebExceptionAdvice}). + */ + @Bean + @ConditionalOnMissingBean(com.github.sonus21.rqueue.repository.MessageBrowsingRepository.class) + public com.github.sonus21.rqueue.repository.MessageBrowsingRepository + natsMessageBrowsingRepository( + JetStreamManagement jetStreamManagement, RqueueNatsProperties props) { + return new com.github.sonus21.rqueue.nats.repository.NatsMessageBrowsingRepository( + jetStreamManagement, toBrokerConfig(props)); + } + + static RqueueNatsConfig toBrokerConfig(RqueueNatsProperties p) { + RqueueNatsConfig cfg = RqueueNatsConfig.defaults(); + cfg.setStreamPrefix(p.getNaming().getStreamPrefix()); + cfg.setSubjectPrefix(p.getNaming().getSubjectPrefix()); + cfg.setDlqStreamSuffix(p.getNaming().getDlqSuffix()); + cfg.setAutoCreateStreams(p.isAutoCreateStreams()); + cfg.setAutoCreateConsumers(p.isAutoCreateConsumers()); + cfg.setAutoCreateDlqStream(p.isAutoCreateDlqStream()); + cfg.setDefaultFetchWait(p.getConsumer().getFetchWait()); + + RqueueNatsConfig.StreamDefaults sd = new RqueueNatsConfig.StreamDefaults(); + sd.setReplicas(p.getStream().getReplicas()); + sd.setStorage( + "MEMORY".equalsIgnoreCase(p.getStream().getStorage()) + ? io.nats.client.api.StorageType.Memory + : io.nats.client.api.StorageType.File); + sd.setRetention( + "WORKQUEUE".equalsIgnoreCase(p.getStream().getRetention()) + ? io.nats.client.api.RetentionPolicy.WorkQueue + : "INTEREST".equalsIgnoreCase(p.getStream().getRetention()) + ? io.nats.client.api.RetentionPolicy.Interest + : io.nats.client.api.RetentionPolicy.Limits); + sd.setMaxMsgs(p.getStream().getMaxMessages()); + sd.setMaxBytes(p.getStream().getMaxBytes()); + sd.setMaxAge(p.getStream().getMaxAge()); + if (p.getStream().getDuplicateWindow() != null) { + sd.setDuplicateWindow(p.getStream().getDuplicateWindow()); + } + cfg.setStreamDefaults(sd); + + RqueueNatsConfig.ConsumerDefaults cd = new RqueueNatsConfig.ConsumerDefaults(); + cd.setAckWait(p.getConsumer().getAckWait()); + cd.setMaxDeliver(p.getConsumer().getMaxDeliver()); + cd.setMaxAckPending(p.getConsumer().getMaxAckPending()); + cfg.setConsumerDefaults(cd); + return cfg; + } +} diff --git a/rqueue-spring-boot-starter/src/main/java/com/github/sonus21/rqueue/spring/boot/RqueueNatsListenerAutoConfig.java b/rqueue-spring-boot-starter/src/main/java/com/github/sonus21/rqueue/spring/boot/RqueueNatsListenerAutoConfig.java new file mode 100644 index 000000000..980dbebf8 --- /dev/null +++ b/rqueue-spring-boot-starter/src/main/java/com/github/sonus21/rqueue/spring/boot/RqueueNatsListenerAutoConfig.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ +package com.github.sonus21.rqueue.spring.boot; + +import com.github.sonus21.rqueue.config.RqueueConfig; +import com.github.sonus21.rqueue.nats.internal.NatsProvisioner; +import com.github.sonus21.rqueue.nats.kv.NatsKvBucketValidator; +import com.github.sonus21.rqueue.nats.worker.NatsWorkerRegistryStore; +import com.github.sonus21.rqueue.worker.RqueueWorkerRegistry; +import com.github.sonus21.rqueue.worker.RqueueWorkerRegistryImpl; +import com.github.sonus21.rqueue.worker.WorkerRegistryStore; +import io.nats.client.Connection; +import io.nats.client.JetStream; +import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.boot.autoconfigure.AutoConfigureAfter; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.DependsOn; + +/** + * Post-listener auto-configuration that wires the NATS-backed worker registry after + * {@link RqueueListenerAutoConfig} has run and the {@link RqueueConfig} bean is available. + * + *

{@link RqueueNatsAutoConfig} must run before the listener config so it can supply + * the {@link com.github.sonus21.rqueue.core.spi.MessageBroker} bean in time. However, the worker + * registry depends on {@link RqueueConfig}, which is only created by + * {@link RqueueListenerAutoConfig}. Splitting these two concerns into two auto-configs — one + * before and one after — breaks the ordering deadlock that caused + * {@code @ConditionalOnBean(RqueueConfig.class)} to always evaluate false. + */ +@AutoConfiguration +@AutoConfigureAfter(RqueueListenerAutoConfig.class) +@ConditionalOnClass(JetStream.class) +@ConditionalOnProperty(name = "rqueue.backend", havingValue = "nats") +public class RqueueNatsListenerAutoConfig { + + /** + * NATS KV-backed store that persists worker registration entries. + * + *

{@code @DependsOn("natsKvBucketValidator")} ensures the KV bucket exists before the store + * tries to bind to it. + */ + @Bean + @ConditionalOnMissingBean(WorkerRegistryStore.class) + @DependsOn("natsKvBucketValidator") + public WorkerRegistryStore natsWorkerRegistryStore( + NatsProvisioner natsProvisioner, com.github.sonus21.rqueue.serdes.RqueueSerDes rqueueSerDes) { + return new NatsWorkerRegistryStore(natsProvisioner, rqueueSerDes); + } + + /** + * Worker registry backed by NATS KV. {@link RqueueConfig} is guaranteed to be present here + * because this auto-config runs after {@link RqueueListenerAutoConfig}. + */ + @Bean + @ConditionalOnMissingBean(RqueueWorkerRegistry.class) + public RqueueWorkerRegistry natsRqueueWorkerRegistry( + RqueueConfig rqueueConfig, WorkerRegistryStore workerRegistryStore) { + return new RqueueWorkerRegistryImpl(rqueueConfig, workerRegistryStore); + } + + /** + * Guard bean that validates KV buckets. Defined here so it is available for + * {@code @DependsOn("natsKvBucketValidator")} references within this config even when the + * primary definition in {@link RqueueNatsAutoConfig} was conditionally skipped. + * + *

In practice the primary definition always wins; this is a fallback safety net. + */ + @Bean + @ConditionalOnMissingBean(NatsKvBucketValidator.class) + public NatsKvBucketValidator natsKvBucketValidatorFallback( + Connection connection, RqueueNatsProperties props) { + return new NatsKvBucketValidator(connection, props.isAutoCreateKvBuckets()); + } +} diff --git a/rqueue-spring-boot-starter/src/main/java/com/github/sonus21/rqueue/spring/boot/RqueueNatsProperties.java b/rqueue-spring-boot-starter/src/main/java/com/github/sonus21/rqueue/spring/boot/RqueueNatsProperties.java new file mode 100644 index 000000000..be28d3e7b --- /dev/null +++ b/rqueue-spring-boot-starter/src/main/java/com/github/sonus21/rqueue/spring/boot/RqueueNatsProperties.java @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ +package com.github.sonus21.rqueue.spring.boot; + +import java.time.Duration; +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** Configuration properties for the optional NATS / JetStream backend. */ +@Getter +@Setter +@ConfigurationProperties(prefix = "rqueue.nats") +public class RqueueNatsProperties { + + private Connection connection = new Connection(); + private Stream stream = new Stream(); + private Consumer consumer = new Consumer(); + private Naming naming = new Naming(); + private boolean autoCreateStreams = true; + private boolean autoCreateConsumers = true; + private boolean autoCreateDlqStream = false; + /** + * When {@code true} (default), each NATS-backed store / dao lazily creates its KV bucket on + * first use. When {@code false}, {@code NatsKvBucketValidator} verifies at startup that every + * required bucket already exists and aborts boot with a clear error if any are missing — for + * deployments where the application credentials lack {@code create} permission on the + * JetStream account. See the "NATS backend" section in the README for the bucket list and + * pre-create commands. + */ + private boolean autoCreateKvBuckets = true; + + @Getter + @Setter + public static class Connection { + private String url; + private String credentialsPath; + private String username; + private String password; + private String token; + private boolean tls; + private String connectionName; + private Duration connectTimeout; + private Duration reconnectWait; + private int maxReconnects = -1; + private Duration pingInterval; + } + + @Getter + @Setter + public static class Stream { + private int replicas = 1; + private String storage = "FILE"; + private String retention = "LIMITS"; + private Duration maxAge = Duration.ofDays(14); + private long maxBytes = -1; + private long maxMessages = -1; + private String discardPolicy = "OLD"; + private Duration duplicateWindow = Duration.ofMinutes(2); + } + + @Getter + @Setter + public static class Consumer { + private Duration ackWait = Duration.ofSeconds(30); + private long maxDeliver = 3; + private long maxAckPending = 1000; + private Duration fetchWait = Duration.ofSeconds(2); + } + + @Getter + @Setter + public static class Naming { + private String streamPrefix = "rqueue-js-"; + private String subjectPrefix = "rqueue.js."; + private String dlqSuffix = "-dlq"; + } +} diff --git a/rqueue-spring-boot-starter/src/main/java/com/github/sonus21/rqueue/spring/boot/RqueueRedisConfigImportSelector.java b/rqueue-spring-boot-starter/src/main/java/com/github/sonus21/rqueue/spring/boot/RqueueRedisConfigImportSelector.java new file mode 100644 index 000000000..18f35b4e8 --- /dev/null +++ b/rqueue-spring-boot-starter/src/main/java/com/github/sonus21/rqueue/spring/boot/RqueueRedisConfigImportSelector.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ +package com.github.sonus21.rqueue.spring.boot; + +import org.springframework.context.annotation.ImportSelector; +import org.springframework.core.type.AnnotationMetadata; +import org.springframework.util.ClassUtils; + +/** + * Conditionally imports {@code RqueueRedisListenerConfig} only when the {@code rqueue-redis} module + * is on the classpath. A direct {@code @Import(RqueueRedisListenerConfig.class)} would cause Spring + * to read the class bytecode before any {@code @Conditional} can fire, throwing + * {@code FileNotFoundException} when the module is excluded. + */ +public class RqueueRedisConfigImportSelector implements ImportSelector { + + private static final String REDIS_CONFIG_CLASS = + "com.github.sonus21.rqueue.redis.config.RqueueRedisListenerConfig"; + + @Override + public String[] selectImports(AnnotationMetadata importingClassMetadata) { + if (ClassUtils.isPresent(REDIS_CONFIG_CLASS, getClass().getClassLoader())) { + return new String[] {REDIS_CONFIG_CLASS}; + } + return new String[0]; + } +} diff --git a/rqueue-spring-boot-starter/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/rqueue-spring-boot-starter/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports index e82e0d5ec..803e4d521 100644 --- a/rqueue-spring-boot-starter/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports +++ b/rqueue-spring-boot-starter/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -1,2 +1,4 @@ com.github.sonus21.rqueue.spring.boot.RqueueListenerAutoConfig -com.github.sonus21.rqueue.spring.boot.RqueueMetricsAutoConfig \ No newline at end of file +com.github.sonus21.rqueue.spring.boot.RqueueMetricsAutoConfig +com.github.sonus21.rqueue.spring.boot.RqueueNatsAutoConfig +com.github.sonus21.rqueue.spring.boot.RqueueNatsListenerAutoConfig diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/RqueueNatsAutoConfigTest.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/RqueueNatsAutoConfigTest.java new file mode 100644 index 000000000..7b7e3f0d1 --- /dev/null +++ b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/RqueueNatsAutoConfigTest.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ +package com.github.sonus21.rqueue.spring.boot; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; + +import com.github.sonus21.rqueue.core.spi.MessageBroker; +import com.github.sonus21.rqueue.nats.js.JetStreamMessageBroker; +import io.nats.client.Connection; +import io.nats.client.JetStream; +import io.nats.client.JetStreamManagement; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.springframework.boot.autoconfigure.AutoConfigurations; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Tag("unit") +@Tag("nats") +class RqueueNatsAutoConfigTest { + + private final ApplicationContextRunner runner = new ApplicationContextRunner() + .withConfiguration(AutoConfigurations.of(RqueueNatsAutoConfig.class)); + + @Test + void doesNotWireBrokerWhenPropertyMissing() { + runner.run(ctx -> assertThat(ctx).doesNotHaveBean(MessageBroker.class)); + } + + @Test + void wiresJetStreamBrokerWhenPropertySetAndJnatsPresent() { + runner + .withPropertyValues("rqueue.backend=nats") + .withUserConfiguration(MockNatsConfig.class) + .run(ctx -> { + assertThat(ctx).hasSingleBean(MessageBroker.class); + assertThat(ctx.getBean(MessageBroker.class)).isInstanceOf(JetStreamMessageBroker.class); + }); + } + + @Test + void doesNotWireWhenBackendSetToOtherValue() { + runner + .withPropertyValues("rqueue.backend=redis") + .withUserConfiguration(MockNatsConfig.class) + .run(ctx -> assertThat(ctx).doesNotHaveBean(MessageBroker.class)); + } + + @Configuration + static class MockNatsConfig { + @Bean + Connection natsConnection() { + return mock(Connection.class); + } + + @Bean + JetStream jetStream() { + return mock(JetStream.class); + } + + @Bean + JetStreamManagement jetStreamManagement() { + return mock(JetStreamManagement.class); + } + } +} diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/application/ApplicationListenerDisabled.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/application/ApplicationListenerDisabled.java index 8175834ba..c5d65755b 100644 --- a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/application/ApplicationListenerDisabled.java +++ b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/application/ApplicationListenerDisabled.java @@ -67,7 +67,7 @@ public RqueueMessageListenerContainer rqueueMessageListenerContainer( } return new RqueueMessageListenerContainer(rqueueMessageHandler, rqueueMessageTemplate) { @Override - protected void startQueue(String queueName, QueueDetail queueDetail) {} + protected void startQueue(String pollerKey, QueueDetail queueDetail) {} }; } } diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/integration/AbstractNatsBootIT.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/integration/AbstractNatsBootIT.java new file mode 100644 index 000000000..92cd39df1 --- /dev/null +++ b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/integration/AbstractNatsBootIT.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ +package com.github.sonus21.rqueue.spring.boot.integration; + +import org.junit.jupiter.api.BeforeAll; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; + +/** + * Common Testcontainers + dynamic-property boilerplate for NATS-backed end-to-end tests. + * + *

Mirrors the existing Redis test pattern (see {@code RedisRunning} / {@code REDIS_RUNNING}): + * when the {@code NATS_RUNNING} environment variable is set the tests assume an externally + * managed nats-server is reachable at {@code NATS_URL} (default {@code nats://127.0.0.1:4222}) + * and skip Testcontainers entirely. CI sets {@code NATS_RUNNING=true} after starting nats-server + * via apt; local dev leaves it unset and falls back to Testcontainers, which itself skips + * gracefully when Docker isn't available. + * + *

Subclasses declare their own {@code @SpringBootApplication} test config (typically excluding + * Redis auto-config, see {@link NatsBackendEndToEndIT} for the reference pattern) and any + * {@code @RqueueListener} beans they need. + */ +@Testcontainers(disabledWithoutDocker = true) +abstract class AbstractNatsBootIT { + + static final boolean USE_EXTERNAL_NATS = System.getenv("NATS_RUNNING") != null; + + static final String EXTERNAL_NATS_URL = + System.getenv().getOrDefault("NATS_URL", "nats://127.0.0.1:4222"); + + static GenericContainer NATS; + + @BeforeAll + static void startNats() { + if (USE_EXTERNAL_NATS || NATS != null) { + return; + } + NATS = new GenericContainer<>(DockerImageName.parse("nats:2.10-alpine")) + .withCommand("-js") + .withExposedPorts(4222) + .waitingFor(Wait.forLogMessage(".*Server is ready.*\\n", 1)); + NATS.start(); + Runtime.getRuntime().addShutdownHook(new Thread(NATS::stop)); + } + + @DynamicPropertySource + static void natsProps(DynamicPropertyRegistry r) { + if (USE_EXTERNAL_NATS) { + r.add("rqueue.nats.connection.url", () -> EXTERNAL_NATS_URL); + } else { + r.add("rqueue.nats.connection.url", () -> { + startNats(); + return "nats://" + NATS.getHost() + ":" + NATS.getMappedPort(4222); + }); + } + } +} diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/integration/NatsBackendEndToEndIT.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/integration/NatsBackendEndToEndIT.java new file mode 100644 index 000000000..55d53780b --- /dev/null +++ b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/integration/NatsBackendEndToEndIT.java @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ +package com.github.sonus21.rqueue.spring.boot.integration; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.github.sonus21.rqueue.annotation.RqueueListener; +import com.github.sonus21.rqueue.core.RqueueMessageEnqueuer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.data.redis.autoconfigure.DataRedisAutoConfiguration; +import org.springframework.boot.data.redis.autoconfigure.DataRedisReactiveAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Import; +import org.springframework.stereotype.Component; + +/** + * End-to-end integration test wiring a Spring Boot application against a NATS JetStream + * instance via {@code rqueue.backend=nats}, an {@link RqueueListener}, and the default + * {@link RqueueMessageEnqueuer}. It exercises the full intended path: + * + * 

+ *   Enqueue -> JetStreamMessageBroker.enqueue -> JetStream stream
+ *           -> BrokerMessagePoller.pop -> @RqueueListener invocation -> broker.ack
+ *
+ * + *

The NATS instance is supplied by {@link AbstractNatsBootIT}: when {@code NATS_RUNNING=true} + * (CI), the test connects to a locally running nats-server; otherwise it falls back to a + * Testcontainers-managed container, which itself skips gracefully without Docker. + * + *

Boots without any Redis at all: every Redis-shaped bean (config DAOs, dashboard controllers, + * pub/sub channel, schedulers) is gated by {@code @Conditional(RedisBackendCondition.class)} and + * stays out of the context when {@code rqueue.backend=nats}. {@code DataRedisAutoConfiguration} + * is excluded so Spring Boot doesn't try to wire a Lettuce client either. + */ +@SpringBootTest( + classes = NatsBackendEndToEndIT.TestApp.class, + properties = {"rqueue.backend=nats"}) +@Tag("nats") +class NatsBackendEndToEndIT extends AbstractNatsBootIT { + + @Autowired + RqueueMessageEnqueuer enqueuer; + + @Autowired + TestListener listener; + + @Test + void enqueueIsReceivedByListener() throws Exception { + for (int i = 0; i < 5; i++) { + enqueuer.enqueue("e2e-test", "payload-" + i); + } + assertThat(listener.latch.await(20, TimeUnit.SECONDS)).isTrue(); + assertThat(listener.received) + .containsExactlyInAnyOrder("payload-0", "payload-1", "payload-2", "payload-3", "payload-4"); + } + + @SpringBootApplication( + exclude = {DataRedisAutoConfiguration.class, DataRedisReactiveAutoConfiguration.class}) + @Import(TestListener.class) + static class TestApp {} + + @Component + static class TestListener { + final CountDownLatch latch = new CountDownLatch(5); + final List received = Collections.synchronizedList(new ArrayList<>()); + + @RqueueListener(value = "e2e-test") + void onMessage(String payload) { + received.add(payload); + latch.countDown(); + } + } +} diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/integration/NatsConcurrencyE2EIT.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/integration/NatsConcurrencyE2EIT.java new file mode 100644 index 000000000..3cdda5eb0 --- /dev/null +++ b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/integration/NatsConcurrencyE2EIT.java @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ +package com.github.sonus21.rqueue.spring.boot.integration; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.github.sonus21.rqueue.annotation.RqueueListener; +import com.github.sonus21.rqueue.core.RqueueMessageEnqueuer; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.data.redis.autoconfigure.DataRedisAutoConfiguration; +import org.springframework.boot.data.redis.autoconfigure.DataRedisReactiveAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Import; +import org.springframework.stereotype.Component; + +/** + * End-to-end test confirming that {@code @RqueueListener(concurrency=...)} actually runs more + * than one handler invocation in parallel against the NATS backend. We don't assert an exact + * parallelism value because JetStream prefetch + thread scheduling makes that flaky; observing + * any parallelism > 1 is enough proof the concurrency knob is wired through to a pull + * subscription with multiple poller threads. + */ +@SpringBootTest( + classes = NatsConcurrencyE2EIT.TestApp.class, + properties = {"rqueue.backend=nats"}) +@Tag("nats") +class NatsConcurrencyE2EIT extends AbstractNatsBootIT { + + @Autowired + RqueueMessageEnqueuer enqueuer; + + @Autowired + ConcurrencyListener listener; + + @Test + void parallelInvocationsAreObserved() throws Exception { + for (int i = 0; i < 30; i++) { + enqueuer.enqueue("conc-e2e", "msg-" + i); + } + assertThat(listener.latch.await(45, TimeUnit.SECONDS)).isTrue(); + assertThat(listener.maxParallel.get()) + .as("at least 2 concurrent invocations should have been observed") + .isGreaterThanOrEqualTo(2); + } + + @SpringBootApplication( + exclude = {DataRedisAutoConfiguration.class, DataRedisReactiveAutoConfiguration.class}) + @Import(ConcurrencyListener.class) + static class TestApp {} + + @Component + static class ConcurrencyListener { + final CountDownLatch latch = new CountDownLatch(30); + final AtomicInteger active = new AtomicInteger(); + final AtomicInteger maxParallel = new AtomicInteger(); + + @RqueueListener(value = "conc-e2e", concurrency = "3") + void onMessage(String payload) throws InterruptedException { + int now = active.incrementAndGet(); + maxParallel.updateAndGet(curr -> Math.max(curr, now)); + try { + Thread.sleep(200L); + } finally { + active.decrementAndGet(); + latch.countDown(); + } + } + } +} diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/integration/NatsConsumerNameOverrideE2EIT.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/integration/NatsConsumerNameOverrideE2EIT.java new file mode 100644 index 000000000..e47b68967 --- /dev/null +++ b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/integration/NatsConsumerNameOverrideE2EIT.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ +package com.github.sonus21.rqueue.spring.boot.integration; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.github.sonus21.rqueue.annotation.RqueueListener; +import com.github.sonus21.rqueue.core.RqueueMessageEnqueuer; +import io.nats.client.JetStreamManagement; +import io.nats.client.api.ConsumerInfo; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.data.redis.autoconfigure.DataRedisAutoConfiguration; +import org.springframework.boot.data.redis.autoconfigure.DataRedisReactiveAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Import; +import org.springframework.stereotype.Component; + +/** + * Verifies that {@code @RqueueListener(consumerName="...")} causes the JetStream durable consumer + * to be created with that exact name (rather than the auto-derived + * {@code rqueue--#} form). After a message round-trips successfully, we + * query the broker via {@link JetStreamManagement#getConsumerInfo} and assert the override is + * present. + */ +@SpringBootTest( + classes = NatsConsumerNameOverrideE2EIT.TestApp.class, + properties = {"rqueue.backend=nats"}) +@Tag("nats") +class NatsConsumerNameOverrideE2EIT extends AbstractNatsBootIT { + + @Autowired + RqueueMessageEnqueuer enqueuer; + + @Autowired + CustomConsumerListener listener; + + @Autowired + JetStreamManagement jsm; + + @Test + void overriddenConsumerNameIsRegisteredOnTheStream() throws Exception { + enqueuer.enqueue("custom-consumer", "hello"); + assertThat(listener.latch.await(20, TimeUnit.SECONDS)).isTrue(); + + ConsumerInfo info = jsm.getConsumerInfo("rqueue-js-custom-consumer", "my-custom-consumer"); + assertThat(info).isNotNull(); + assertThat(info.getName()).isEqualTo("my-custom-consumer"); + } + + @SpringBootApplication( + exclude = {DataRedisAutoConfiguration.class, DataRedisReactiveAutoConfiguration.class}) + @Import(CustomConsumerListener.class) + static class TestApp {} + + @Component + static class CustomConsumerListener { + final CountDownLatch latch = new CountDownLatch(1); + + @RqueueListener(value = "custom-consumer", consumerName = "my-custom-consumer") + void onMessage(String payload) { + latch.countDown(); + } + } +} diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/integration/NatsMultipleListenersOnSameQueueE2EIT.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/integration/NatsMultipleListenersOnSameQueueE2EIT.java new file mode 100644 index 000000000..e7dddcbe4 --- /dev/null +++ b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/integration/NatsMultipleListenersOnSameQueueE2EIT.java @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ +package com.github.sonus21.rqueue.spring.boot.integration; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.github.sonus21.rqueue.annotation.RqueueListener; +import com.github.sonus21.rqueue.core.RqueueMessageEnqueuer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.data.redis.autoconfigure.DataRedisAutoConfiguration; +import org.springframework.boot.data.redis.autoconfigure.DataRedisReactiveAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.stereotype.Component; + +/** + * Two {@code @RqueueListener} methods on the same queue should each receive every message + * published to it (independent durable consumers, fan-out semantics). Currently disabled because + * the default JetStream stream retention is {@code WorkQueue}, which permits only one + * filter-overlapping consumer at a time and deletes a message after the first ack — so true + * fan-out is not possible with the v1 broker defaults. To enable this test, the broker would + * need to switch to {@code Limits} or {@code Interest} retention for queues with multiple + * listeners. + */ +@SpringBootTest( + classes = NatsMultipleListenersOnSameQueueE2EIT.TestApp.class, + properties = {"rqueue.backend=nats"}) +@Tag("nats") +@Disabled("Default JetStream retention=WorkQueue prevents true fan-out across multiple consumers; " + + "enable once retention is configurable per queue or defaulted to Limits/Interest.") +class NatsMultipleListenersOnSameQueueE2EIT extends AbstractNatsBootIT { + + @Autowired + RqueueMessageEnqueuer enqueuer; + + @Autowired + ListenerOne one; + + @Autowired + ListenerTwo two; + + @Test + void bothListenersReceiveAllMessages() throws Exception { + for (int i = 0; i < 5; i++) { + enqueuer.enqueue("multi", "fan-" + i); + } + assertThat(one.latch.await(20, TimeUnit.SECONDS)).isTrue(); + assertThat(two.latch.await(20, TimeUnit.SECONDS)).isTrue(); + assertThat(one.received).hasSize(5); + assertThat(two.received).hasSize(5); + } + + @SpringBootApplication( + exclude = {DataRedisAutoConfiguration.class, DataRedisReactiveAutoConfiguration.class}) + static class TestApp {} + + @Component + static class ListenerOne { + final CountDownLatch latch = new CountDownLatch(5); + final List received = Collections.synchronizedList(new ArrayList<>()); + + @RqueueListener(value = "multi") + void onMessage(String payload) { + received.add(payload); + latch.countDown(); + } + } + + @Component + static class ListenerTwo { + final CountDownLatch latch = new CountDownLatch(5); + final List received = Collections.synchronizedList(new ArrayList<>()); + + @RqueueListener(value = "multi") + void onMessage(String payload) { + received.add(payload); + latch.countDown(); + } + } +} diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/integration/NatsPriorityQueuesE2EIT.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/integration/NatsPriorityQueuesE2EIT.java new file mode 100644 index 000000000..6640ac9d8 --- /dev/null +++ b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/integration/NatsPriorityQueuesE2EIT.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ +package com.github.sonus21.rqueue.spring.boot.integration; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.github.sonus21.rqueue.annotation.RqueueListener; +import com.github.sonus21.rqueue.core.RqueueMessageEnqueuer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.data.redis.autoconfigure.DataRedisAutoConfiguration; +import org.springframework.boot.data.redis.autoconfigure.DataRedisReactiveAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Import; +import org.springframework.stereotype.Component; + +/** + * Verifies queue-level priority on the NATS backend: a single listener with + * {@code priority="high=10,low=1"} consumes from two internal sub-queues + * ({@code pq_high} and {@code pq_low}) and the producer sends to each via + * {@link RqueueMessageEnqueuer#enqueueWithPriority}. We assert all 10 messages are + * received and that 5 messages with payload prefix "high-" and 5 with "low-" arrive. + */ +@SpringBootTest( + classes = NatsPriorityQueuesE2EIT.TestApp.class, + properties = {"rqueue.backend=nats"}) +@Tag("nats") +class NatsPriorityQueuesE2EIT extends AbstractNatsBootIT { + + @Autowired + RqueueMessageEnqueuer enqueuer; + + @Autowired + PriorityListener listener; + + @Test + void messagesEnqueuedAtBothPrioritiesAreReceived() throws Exception { + for (int i = 0; i < 5; i++) { + enqueuer.enqueueWithPriority("pq", "high", "high-" + i); + enqueuer.enqueueWithPriority("pq", "low", "low-" + i); + } + assertThat(listener.latch.await(30, TimeUnit.SECONDS)).isTrue(); + + long highCount = + listener.received.stream().filter(s -> s.startsWith("high-")).count(); + long lowCount = listener.received.stream().filter(s -> s.startsWith("low-")).count(); + assertThat(highCount).isEqualTo(5); + assertThat(lowCount).isEqualTo(5); + } + + @SpringBootApplication( + exclude = {DataRedisAutoConfiguration.class, DataRedisReactiveAutoConfiguration.class}) + @Import(PriorityListener.class) + static class TestApp {} + + @Component + static class PriorityListener { + final CountDownLatch latch = new CountDownLatch(10); + final List received = Collections.synchronizedList(new ArrayList<>()); + + @RqueueListener(value = "pq", priority = "high=10,low=1") + void onMessage(String payload) { + received.add(payload); + latch.countDown(); + } + } +} diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/integration/NatsReactiveEnqueueE2EIT.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/integration/NatsReactiveEnqueueE2EIT.java new file mode 100644 index 000000000..596b5615d --- /dev/null +++ b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/integration/NatsReactiveEnqueueE2EIT.java @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ +package com.github.sonus21.rqueue.spring.boot.integration; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.github.sonus21.rqueue.annotation.RqueueListener; +import com.github.sonus21.rqueue.core.ReactiveRqueueMessageEnqueuer; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.data.redis.autoconfigure.DataRedisAutoConfiguration; +import org.springframework.boot.data.redis.autoconfigure.DataRedisReactiveAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Import; +import org.springframework.stereotype.Component; +import reactor.core.publisher.Flux; + +/** + * Verifies the reactive producer path on the NATS backend: enqueueing 5 messages via + * {@link ReactiveRqueueMessageEnqueuer} (subscribed via {@link Flux#merge}) and confirming a + * synchronous {@code @RqueueListener} on the same queue receives all 5. + */ +@SpringBootTest( + classes = NatsReactiveEnqueueE2EIT.TestApp.class, + properties = {"rqueue.backend=nats", "rqueue.reactive.enabled=true"}) +@Tag("nats") +class NatsReactiveEnqueueE2EIT extends AbstractNatsBootIT { + + @Autowired + ReactiveRqueueMessageEnqueuer reactiveEnqueuer; + + @Autowired + ReactiveListener listener; + + @Test + void reactivelyEnqueuedMessagesAreReceivedByListener() throws Exception { + List> publishers = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + publishers.add(reactiveEnqueuer.enqueue("reactive-e2e", "rx-" + i)); + } + List ids = Flux.merge(publishers).collectList().block(Duration.ofSeconds(15)); + assertThat(ids).hasSize(5); + + assertThat(listener.latch.await(20, TimeUnit.SECONDS)).isTrue(); + assertThat(listener.received).containsExactlyInAnyOrder("rx-0", "rx-1", "rx-2", "rx-3", "rx-4"); + } + + @SpringBootApplication( + exclude = {DataRedisAutoConfiguration.class, DataRedisReactiveAutoConfiguration.class}) + @Import(ReactiveListener.class) + static class TestApp {} + + @Component + static class ReactiveListener { + final CountDownLatch latch = new CountDownLatch(5); + final List received = Collections.synchronizedList(new ArrayList<>()); + + @RqueueListener(value = "reactive-e2e") + void onMessage(String payload) { + received.add(payload); + latch.countDown(); + } + } +} diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/integration/NatsRetryAndDlqE2EIT.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/integration/NatsRetryAndDlqE2EIT.java new file mode 100644 index 000000000..123798aec --- /dev/null +++ b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/integration/NatsRetryAndDlqE2EIT.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ +package com.github.sonus21.rqueue.spring.boot.integration; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.github.sonus21.rqueue.annotation.RqueueListener; +import com.github.sonus21.rqueue.core.RqueueMessageEnqueuer; +import io.nats.client.JetStreamManagement; +import io.nats.client.api.StreamInfo; +import java.time.Duration; +import java.util.concurrent.atomic.AtomicInteger; +import org.awaitility.Awaitility; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.data.redis.autoconfigure.DataRedisAutoConfiguration; +import org.springframework.boot.data.redis.autoconfigure.DataRedisReactiveAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.stereotype.Component; + +/** + * After a handler exhausts {@code numRetries}, JetStream emits a max-deliveries advisory and the + * broker's {@code installDeadLetterBridge} dispatcher republishes the payload onto the DLQ + * stream. Currently disabled because {@link + * com.github.sonus21.rqueue.spring.boot.RqueueNatsAutoConfig} does not yet invoke + * {@code JetStreamMessageBroker.installDeadLetterBridge(...)} during container start, so dead- + * lettered messages never reach the DLQ stream. Enable this test once that wiring is added. + */ +@SpringBootTest( + classes = NatsRetryAndDlqE2EIT.TestApp.class, + properties = {"rqueue.backend=nats"}) +@Tag("nats") +@Disabled( + "DLQ bridge wiring (JetStreamMessageBroker.installDeadLetterBridge) is not yet invoked by " + + "RqueueNatsAutoConfig; enable once the container start path provisions the advisory " + + "dispatcher per queue.") +class NatsRetryAndDlqE2EIT extends AbstractNatsBootIT { + + @Autowired + RqueueMessageEnqueuer enqueuer; + + @Autowired + FailingListener listener; + + @Autowired + JetStreamManagement jsm; + + @Test + void exhaustedMessageLandsOnDlqStream() { + enqueuer.enqueue("failing", "boom"); + + Awaitility.await().atMost(Duration.ofSeconds(60)).until(() -> listener.attempts.get() >= 2); + + Awaitility.await().atMost(Duration.ofSeconds(30)).untilAsserted(() -> { + StreamInfo dlq = jsm.getStreamInfo("rqueue-js-failing-dlq"); + assertThat(dlq.getStreamState().getMsgCount()).isGreaterThanOrEqualTo(1); + }); + } + + @SpringBootApplication( + exclude = {DataRedisAutoConfiguration.class, DataRedisReactiveAutoConfiguration.class}) + static class TestApp {} + + @Component + static class FailingListener { + final AtomicInteger attempts = new AtomicInteger(); + + @RqueueListener(value = "failing", numRetries = "2") + void onMessage(String payload) { + attempts.incrementAndGet(); + throw new RuntimeException("simulated failure for payload=" + payload); + } + } +} diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/CustomMessageConverterTest.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/CustomMessageConverterTest.java index af87e9b0e..4135815ef 100644 --- a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/CustomMessageConverterTest.java +++ b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/CustomMessageConverterTest.java @@ -21,12 +21,12 @@ import com.github.sonus21.rqueue.core.RqueueMessage; import com.github.sonus21.rqueue.exception.TimedOutException; +import com.github.sonus21.rqueue.serdes.SerializationUtils; import com.github.sonus21.rqueue.spring.boot.application.ApplicationWithCustomMessageConverter; import com.github.sonus21.rqueue.spring.boot.tests.SpringBootIntegrationTest; import com.github.sonus21.rqueue.test.dto.Job; import com.github.sonus21.rqueue.test.tests.BasicListenerTest; import com.github.sonus21.rqueue.utils.Constants; -import com.github.sonus21.rqueue.utils.SerializationUtils; import com.github.sonus21.rqueue.utils.TimeoutUtils; import java.util.ArrayList; import java.util.List; diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/PauseUnpauseTest.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/PauseUnpauseTest.java index 9b206d64a..a82146e31 100644 --- a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/PauseUnpauseTest.java +++ b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/PauseUnpauseTest.java @@ -24,6 +24,7 @@ import com.github.sonus21.AtomicValueHolder; import com.github.sonus21.rqueue.exception.TimedOutException; import com.github.sonus21.rqueue.models.request.PauseUnpauseQueueRequest; +import com.github.sonus21.rqueue.service.RqueueUtilityService; import com.github.sonus21.rqueue.spring.boot.application.Application; import com.github.sonus21.rqueue.spring.boot.tests.SpringBootIntegrationTest; import com.github.sonus21.rqueue.test.PauseUnpauseEventListener; @@ -31,7 +32,6 @@ import com.github.sonus21.rqueue.test.dto.Notification; import com.github.sonus21.rqueue.utils.Constants; import com.github.sonus21.rqueue.utils.TimeoutUtils; -import com.github.sonus21.rqueue.web.service.RqueueUtilityService; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/unit/RqueueListenerAutoConfigTest.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/unit/RqueueListenerAutoConfigTest.java index aab258bbb..b7e2986ec 100644 --- a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/unit/RqueueListenerAutoConfigTest.java +++ b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/unit/RqueueListenerAutoConfigTest.java @@ -18,8 +18,10 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.when; import com.github.sonus21.TestBase; import com.github.sonus21.rqueue.config.SimpleRqueueListenerContainerFactory; @@ -30,6 +32,8 @@ import com.github.sonus21.rqueue.core.RqueueMessageEnqueuer; import com.github.sonus21.rqueue.core.RqueueMessageTemplate; import com.github.sonus21.rqueue.core.impl.UuidV4RqueueMessageIdGenerator; +import com.github.sonus21.rqueue.core.spi.Capabilities; +import com.github.sonus21.rqueue.core.spi.MessageBroker; import com.github.sonus21.rqueue.listener.RqueueMessageHandler; import com.github.sonus21.rqueue.spring.boot.RqueueListenerAutoConfig; import com.github.sonus21.rqueue.spring.boot.tests.SpringBootUnitTest; @@ -58,6 +62,9 @@ class RqueueListenerAutoConfigTest extends TestBase { @Mock private RqueueMessageHandler rqueueMessageHandler; + @Mock + private MessageBroker messageBroker; + @Mock private RedisConnectionFactory redisConnectionFactory; @@ -77,7 +84,8 @@ public void init() throws IllegalAccessException { @Test void rqueueMessageHandlerDefaultCreation() throws ClassNotFoundException, InstantiationException, IllegalAccessException { - assertNotNull(rqueueMessageAutoConfig.rqueueMessageHandler()); + when(messageBroker.capabilities()).thenReturn(Capabilities.REDIS_DEFAULTS); + assertNotNull(rqueueMessageAutoConfig.rqueueMessageHandler(messageBroker)); } @Test @@ -92,9 +100,10 @@ void rqueueMessageHandlerReused() "com.github.sonus21.rqueue.converter.DefaultMessageConverterProvider", true); FieldUtils.writeField(messageAutoConfig, "simpleRqueueListenerContainerFactory", factory, true); + when(messageBroker.capabilities()).thenReturn(Capabilities.REDIS_DEFAULTS); assertEquals( rqueueMessageHandler.hashCode(), - messageAutoConfig.rqueueMessageHandler().hashCode()); + messageAutoConfig.rqueueMessageHandler(messageBroker).hashCode()); } @Test @@ -102,7 +111,7 @@ void rqueueMessageListenerContainer() throws IllegalAccessException, ClassNotFoundException, InstantiationException { SimpleRqueueListenerContainerFactory factory = new SimpleRqueueListenerContainerFactory(); factory.setMessageConverterProvider(new DefaultMessageConverterProvider()); - factory.setRedisConnectionFactory(redisConnectionFactory); + factory.setMessageBroker(messageBroker); RqueueListenerAutoConfig messageAutoConfig = new RqueueListenerAutoConfig(); FieldUtils.writeField( messageAutoConfig, @@ -110,25 +119,32 @@ void rqueueMessageListenerContainer() "com.github.sonus21.rqueue.converter.DefaultMessageConverterProvider", true); FieldUtils.writeField(messageAutoConfig, "simpleRqueueListenerContainerFactory", factory, true); - messageAutoConfig.rqueueMessageListenerContainer(rqueueMessageHandler); + messageAutoConfig.rqueueMessageListenerContainer(rqueueMessageHandler, messageBroker); assertEquals(factory.getRqueueMessageHandler(null).hashCode(), rqueueMessageHandler.hashCode()); + // The broker must be propagated onto the factory so the container picks it up. + assertSame(messageBroker, factory.getMessageBroker()); } @Test - void rqueueMessageSenderWithMessageTemplate() throws IllegalAccessException { + void rqueueMessageEnqueuerWiresBroker() throws IllegalAccessException { SimpleRqueueListenerContainerFactory factory = new SimpleRqueueListenerContainerFactory(); factory.setMessageConverterProvider(new DefaultMessageConverterProvider()); factory.setRqueueMessageTemplate(messageTemplate); doReturn(new DefaultRqueueMessageConverter()).when(rqueueMessageHandler).getMessageConverter(); RqueueListenerAutoConfig messageAutoConfig = new RqueueListenerAutoConfig(); FieldUtils.writeField(messageAutoConfig, "simpleRqueueListenerContainerFactory", factory, true); - assertNotNull(messageAutoConfig.rqueueMessageEnqueuer( - rqueueMessageHandler, messageTemplate, new UuidV4RqueueMessageIdGenerator())); - assertEquals(factory.getRqueueMessageTemplate().hashCode(), messageTemplate.hashCode()); + + RqueueMessageEnqueuer enqueuer = messageAutoConfig.rqueueMessageEnqueuer( + rqueueMessageHandler, messageTemplate, messageBroker, new UuidV4RqueueMessageIdGenerator()); + + assertNotNull(enqueuer); + // Broker is on the enqueuer (inherited from BaseMessageSender), not on the template — that + // sidesteps the Redis cycle and removes the original NPE class entirely. + assertSame(messageBroker, FieldUtils.readField(enqueuer, "messageBroker", true)); } @Test - void rqueueMessageSenderWithMessageConverters() throws IllegalAccessException { + void rqueueMessageSenderUsesConfiguredMessageConverter() throws IllegalAccessException { MessageConverter messageConverter = new GenericMessageConverter(); MessageConverterProvider messageConverterProvider = () -> messageConverter; SimpleRqueueListenerContainerFactory factory = new SimpleRqueueListenerContainerFactory(); @@ -137,10 +153,8 @@ void rqueueMessageSenderWithMessageConverters() throws IllegalAccessException { factory.setRqueueMessageTemplate(messageTemplate); FieldUtils.writeField(messageAutoConfig, "simpleRqueueListenerContainerFactory", factory, true); doReturn(messageConverter).when(rqueueMessageHandler).getMessageConverter(); - assertNotNull(messageAutoConfig.rqueueMessageEnqueuer( - rqueueMessageHandler, messageTemplate, new UuidV4RqueueMessageIdGenerator())); RqueueMessageEnqueuer messageSender = messageAutoConfig.rqueueMessageEnqueuer( - rqueueMessageHandler, messageTemplate, new UuidV4RqueueMessageIdGenerator()); + rqueueMessageHandler, messageTemplate, messageBroker, new UuidV4RqueueMessageIdGenerator()); MessageConverter converter = messageSender.getMessageConverter(); assertTrue(converter.hashCode() == messageConverter.hashCode()); } diff --git a/rqueue-spring-common-test/build.gradle b/rqueue-spring-common-test/build.gradle index 133fb0875..020298ce2 100644 --- a/rqueue-spring-common-test/build.gradle +++ b/rqueue-spring-common-test/build.gradle @@ -1,5 +1,8 @@ dependencies { implementation project(":rqueue-core") + // Brings in spring-webflux (WebClient), spring-webmvc, jakarta.servlet, Pebble, etc. — all + // of which the common-test base classes (SpringWebTestBase) consume. + api project(":rqueue-web") api project(":rqueue-test-util") // https://mvnrepository.com/artifact/io.micrometer/micrometer-registry-prometheus api "io.micrometer:micrometer-registry-prometheus:${microMeterVersion}" diff --git a/rqueue-spring-common-test/src/main/java/com/github/sonus21/rqueue/test/application/ApplicationBasicConfiguration.java b/rqueue-spring-common-test/src/main/java/com/github/sonus21/rqueue/test/application/ApplicationBasicConfiguration.java index 6138f46e4..e8f519f67 100644 --- a/rqueue-spring-common-test/src/main/java/com/github/sonus21/rqueue/test/application/ApplicationBasicConfiguration.java +++ b/rqueue-spring-common-test/src/main/java/com/github/sonus21/rqueue/test/application/ApplicationBasicConfiguration.java @@ -19,7 +19,7 @@ import com.athaydes.javanna.Javanna; import com.github.sonus21.junit.BootstrapRedis; import com.github.sonus21.junit.RedisBootstrapperBase; -import com.github.sonus21.rqueue.utils.SerializationUtils; +import com.github.sonus21.rqueue.serdes.SerializationUtils; import java.io.IOException; import java.util.HashMap; import javax.sql.DataSource; @@ -90,6 +90,6 @@ public LocalContainerEntityManagerFactoryBean entityManagerFactory(DataSource da @Bean public ObjectMapper objectMapper() { - return SerializationUtils.createObjectMapper(); + return SerializationUtils.getObjectMapper(); } } diff --git a/rqueue-spring-common-test/src/main/java/com/github/sonus21/rqueue/test/common/SpringTestBase.java b/rqueue-spring-common-test/src/main/java/com/github/sonus21/rqueue/test/common/SpringTestBase.java index 29303dbd7..649e065e1 100644 --- a/rqueue-spring-common-test/src/main/java/com/github/sonus21/rqueue/test/common/SpringTestBase.java +++ b/rqueue-spring-common-test/src/main/java/com/github/sonus21/rqueue/test/common/SpringTestBase.java @@ -33,14 +33,14 @@ import com.github.sonus21.rqueue.exception.TimedOutException; import com.github.sonus21.rqueue.listener.QueueDetail; import com.github.sonus21.rqueue.listener.RqueueMessageListenerContainer; -import com.github.sonus21.rqueue.metrics.RqueueQueueMetrics; +import com.github.sonus21.rqueue.metrics.RqueueQueueMetricsProvider; +import com.github.sonus21.rqueue.service.RqueueMessageMetadataService; import com.github.sonus21.rqueue.test.entity.ConsumedMessage; import com.github.sonus21.rqueue.test.service.ConsumedMessageStore; import com.github.sonus21.rqueue.test.service.FailureManager; import com.github.sonus21.rqueue.test.service.RqueueEventListener; import com.github.sonus21.rqueue.utils.StringUtils; import com.github.sonus21.rqueue.utils.TimeoutUtils; -import com.github.sonus21.rqueue.web.service.RqueueMessageMetadataService; import java.time.Duration; import java.time.Instant; import java.util.Collections; @@ -103,7 +103,7 @@ public abstract class SpringTestBase extends TestBase { protected ObjectMapper objectMapper; @Autowired - protected RqueueQueueMetrics rqueueQueueMetrics; + protected RqueueQueueMetricsProvider rqueueQueueMetrics; @Value("${email.queue.name}") protected String emailQueue; diff --git a/rqueue-spring-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MetricTest.java b/rqueue-spring-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MetricTest.java index f8c8661a1..8f6c98709 100644 --- a/rqueue-spring-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MetricTest.java +++ b/rqueue-spring-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MetricTest.java @@ -22,7 +22,7 @@ import com.github.sonus21.rqueue.core.EndpointRegistry; import com.github.sonus21.rqueue.exception.TimedOutException; import com.github.sonus21.rqueue.listener.QueueDetail; -import com.github.sonus21.rqueue.metrics.RqueueQueueMetrics; +import com.github.sonus21.rqueue.metrics.RqueueQueueMetricsProvider; import com.github.sonus21.rqueue.test.common.SpringTestBase; import com.github.sonus21.rqueue.test.dto.Email; import com.github.sonus21.rqueue.test.dto.Job; @@ -40,7 +40,7 @@ public abstract class MetricTest extends SpringTestBase { protected MeterRegistry meterRegistry; @Autowired - protected RqueueQueueMetrics rqueueQueueMetrics; + protected RqueueQueueMetricsProvider rqueueQueueMetrics; protected void verifyScheduledQueueStatus() throws TimedOutException { long maxDelay = 0; diff --git a/rqueue-spring/build.gradle b/rqueue-spring/build.gradle index e58c6dc2a..7c6b2acd1 100644 --- a/rqueue-spring/build.gradle +++ b/rqueue-spring/build.gradle @@ -41,5 +41,18 @@ mavenPublishing { dependencies { api project(":rqueue-core") + // Default-on dashboard module. Consumers who want a headless worker can rqueue-web. + api project(":rqueue-web") + // Redis backend (default). Carries the @Configuration that wires Redis-only @Beans extracted + // from rqueue-core's RqueueListenerBaseConfig. + api project(":rqueue-redis") + + // NATS support is opt-in. RqueueListenerConfig conditionally wires JetStreamMessageBroker + // when rqueue-nats and io.nats:jnats are on the classpath; otherwise the existing Redis + // path is used. Users who want NATS pull rqueue-nats explicitly. + compileOnly project(":rqueue-nats") + compileOnly "io.nats:jnats:${natsVersion}" + testImplementation project(":rqueue-spring-common-test") + testImplementation project(":rqueue-nats") } diff --git a/rqueue-spring/src/main/java/com/github/sonus21/rqueue/spring/EnableRqueue.java b/rqueue-spring/src/main/java/com/github/sonus21/rqueue/spring/EnableRqueue.java index dd3926c3d..29cb7df05 100644 --- a/rqueue-spring/src/main/java/com/github/sonus21/rqueue/spring/EnableRqueue.java +++ b/rqueue-spring/src/main/java/com/github/sonus21/rqueue/spring/EnableRqueue.java @@ -16,6 +16,7 @@ package com.github.sonus21.rqueue.spring; +import com.github.sonus21.rqueue.config.Backend; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @@ -23,17 +24,25 @@ import org.springframework.context.annotation.Import; /** - * This annotation can be used to auto configure Rqueue library by providing some sample - * configuration like by just providing - * {@link org.springframework.data.redis.connection.RedisConnectionFactory}. - * - *

All other beans would be created automatically. Though other components of library can be - * configured as well using - * {@link com.github.sonus21.rqueue.config.SimpleRqueueListenerContainerFactory}. Even it can be - * configured at very fine-grained level by creating all individual beans created in - * {@link RqueueListenerConfig} + * Auto-configure Rqueue when {@link + * org.springframework.data.redis.connection.RedisConnectionFactory} (or, when {@link + * #backend()} is {@link Backend#NATS}, an {@link io.nats.client.Connection}-derived + * {@code MessageBroker}) is available. All other beans are created automatically; further + * customization happens through {@link + * com.github.sonus21.rqueue.config.SimpleRqueueListenerContainerFactory} or by directly defining + * the beans created in {@link RqueueListenerConfig}. */ @Target(ElementType.TYPE) @Retention(RetentionPolicy.RUNTIME) -@Import({RqueueListenerConfig.class}) -public @interface EnableRqueue {} +@Import({RqueueBackendImportSelector.class}) +public @interface EnableRqueue { + + /** + * Backend to wire. Defaults to {@link Backend#REDIS}. Set to {@link Backend#NATS} to import the + * NATS / JetStream listener configuration. The {@code rqueue.backend} property is independently + * read by {@link com.github.sonus21.rqueue.config.RqueueConfig} and remains the source of truth + * for the runtime backend; this attribute controls only which {@code @Configuration} class is + * pulled into the context. + */ + Backend backend() default Backend.REDIS; +} diff --git a/rqueue-spring/src/main/java/com/github/sonus21/rqueue/spring/NatsBackendCondition.java b/rqueue-spring/src/main/java/com/github/sonus21/rqueue/spring/NatsBackendCondition.java new file mode 100644 index 000000000..0d4f3ec4c --- /dev/null +++ b/rqueue-spring/src/main/java/com/github/sonus21/rqueue/spring/NatsBackendCondition.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ +package com.github.sonus21.rqueue.spring; + +import org.springframework.context.annotation.Condition; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.core.type.AnnotatedTypeMetadata; +import org.springframework.util.ClassUtils; + +/** + * Activates the NATS backend wiring when {@code io.nats.client.JetStream} is on the classpath + * and {@code rqueue.backend=nats} is set in the environment. + */ +public class NatsBackendCondition implements Condition { + @Override + public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { + boolean classPresent = + ClassUtils.isPresent("io.nats.client.JetStream", context.getClassLoader()); + String backend = context.getEnvironment().getProperty("rqueue.backend"); + return classPresent && "nats".equalsIgnoreCase(backend); + } +} diff --git a/rqueue-spring/src/main/java/com/github/sonus21/rqueue/spring/RqueueBackendImportSelector.java b/rqueue-spring/src/main/java/com/github/sonus21/rqueue/spring/RqueueBackendImportSelector.java new file mode 100644 index 000000000..54989ccba --- /dev/null +++ b/rqueue-spring/src/main/java/com/github/sonus21/rqueue/spring/RqueueBackendImportSelector.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ +package com.github.sonus21.rqueue.spring; + +import com.github.sonus21.rqueue.config.Backend; +import org.springframework.context.annotation.ImportSelector; +import org.springframework.core.type.AnnotationMetadata; + +/** + * Selects the listener configuration classes to import based on {@link EnableRqueue#backend()}. + * + *

    + *
  • {@link Backend#REDIS} (default) — only {@code RqueueListenerConfig}.
    • + *
  • {@link Backend#NATS} — both {@code RqueueListenerConfig} and {@code + * RqueueNatsListenerConfig}.
    • + *
+ */ +public class RqueueBackendImportSelector implements ImportSelector { + @Override + public String[] selectImports(AnnotationMetadata importingClassMetadata) { + Backend backend = Backend.REDIS; + Object raw = + importingClassMetadata.getAnnotationAttributes(EnableRqueue.class.getName()) == null + ? null + : importingClassMetadata + .getAnnotationAttributes(EnableRqueue.class.getName()) + .get("backend"); + if (raw instanceof Backend b) { + backend = b; + } else if (raw != null) { + try { + backend = Backend.valueOf(raw.toString()); + } catch (IllegalArgumentException ignored) { + // keep default + } + } + if (backend == Backend.NATS) { + return new String[] { + RqueueListenerConfig.class.getName(), RqueueNatsListenerConfig.class.getName() + }; + } + return new String[] {RqueueListenerConfig.class.getName()}; + } +} diff --git a/rqueue-spring/src/main/java/com/github/sonus21/rqueue/spring/RqueueListenerConfig.java b/rqueue-spring/src/main/java/com/github/sonus21/rqueue/spring/RqueueListenerConfig.java index cb92a1709..a221eec75 100644 --- a/rqueue-spring/src/main/java/com/github/sonus21/rqueue/spring/RqueueListenerConfig.java +++ b/rqueue-spring/src/main/java/com/github/sonus21/rqueue/spring/RqueueListenerConfig.java @@ -28,6 +28,7 @@ import com.github.sonus21.rqueue.core.impl.RqueueEndpointManagerImpl; import com.github.sonus21.rqueue.core.impl.RqueueMessageEnqueuerImpl; import com.github.sonus21.rqueue.core.impl.RqueueMessageManagerImpl; +import com.github.sonus21.rqueue.core.spi.MessageBroker; import com.github.sonus21.rqueue.listener.RqueueMessageHandler; import com.github.sonus21.rqueue.listener.RqueueMessageListenerContainer; import com.github.sonus21.rqueue.metrics.QueueCounter; @@ -41,15 +42,24 @@ import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.DependsOn; +import org.springframework.context.annotation.Import; @Configuration -@ComponentScan({"com.github.sonus21.rqueue.web", "com.github.sonus21.rqueue.dao"}) +@ComponentScan({ + "com.github.sonus21.rqueue.web", + "com.github.sonus21.rqueue.dao", + "com.github.sonus21.rqueue.nats", +}) +@Import(RqueueRedisConfigImportSelector.class) public class RqueueListenerConfig extends RqueueListenerBaseConfig { @Bean - public RqueueMessageHandler rqueueMessageHandler() { - return simpleRqueueListenerContainerFactory.getRqueueMessageHandler( - getMessageConverterProvider()); + public RqueueMessageHandler rqueueMessageHandler(MessageBroker messageBroker) { + RqueueMessageHandler handler = + simpleRqueueListenerContainerFactory.getRqueueMessageHandler(getMessageConverterProvider()); + handler.setPrimaryHandlerDispatchEnabled( + messageBroker.capabilities().usesPrimaryHandlerDispatch()); + return handler; } @Bean @@ -69,9 +79,11 @@ public RqueueMessageTemplate rqueueMessageTemplate(RqueueConfig rqueueConfig) { public RqueueMessageManager rqueueMessageManager( RqueueMessageHandler rqueueMessageHandler, RqueueMessageTemplate rqueueMessageTemplate, + MessageBroker messageBroker, RqueueMessageIdGenerator rqueueMessageIdGenerator) { return new RqueueMessageManagerImpl( rqueueMessageTemplate, + messageBroker, rqueueMessageHandler.getMessageConverter(), simpleRqueueListenerContainerFactory.getMessageHeaders(), rqueueMessageIdGenerator); @@ -81,9 +93,11 @@ public RqueueMessageManager rqueueMessageManager( public RqueueEndpointManager rqueueEndpointManager( RqueueMessageHandler rqueueMessageHandler, RqueueMessageTemplate rqueueMessageTemplate, + MessageBroker messageBroker, RqueueMessageIdGenerator rqueueMessageIdGenerator) { return new RqueueEndpointManagerImpl( rqueueMessageTemplate, + messageBroker, rqueueMessageHandler.getMessageConverter(), simpleRqueueListenerContainerFactory.getMessageHeaders(), rqueueMessageIdGenerator); @@ -93,9 +107,11 @@ public RqueueEndpointManager rqueueEndpointManager( public RqueueMessageEnqueuer rqueueMessageEnqueuer( RqueueMessageHandler rqueueMessageHandler, RqueueMessageTemplate rqueueMessageTemplate, + MessageBroker messageBroker, RqueueMessageIdGenerator rqueueMessageIdGenerator) { return new RqueueMessageEnqueuerImpl( rqueueMessageTemplate, + messageBroker, rqueueMessageHandler.getMessageConverter(), simpleRqueueListenerContainerFactory.getMessageHeaders(), rqueueMessageIdGenerator); @@ -120,9 +136,11 @@ public RqueueMetricsCounter rqueueMetricsCounter(RqueueMetricsRegistry rqueueMet public ReactiveRqueueMessageEnqueuer reactiveRqueueMessageEnqueuer( RqueueMessageHandler rqueueMessageHandler, RqueueMessageTemplate rqueueMessageTemplate, + MessageBroker messageBroker, RqueueMessageIdGenerator rqueueMessageIdGenerator) { return new ReactiveRqueueMessageEnqueuerImpl( rqueueMessageTemplate, + messageBroker, rqueueMessageHandler.getMessageConverter(), simpleRqueueListenerContainerFactory.getMessageHeaders(), rqueueMessageIdGenerator); diff --git a/rqueue-spring/src/main/java/com/github/sonus21/rqueue/spring/RqueueNatsListenerConfig.java b/rqueue-spring/src/main/java/com/github/sonus21/rqueue/spring/RqueueNatsListenerConfig.java new file mode 100644 index 000000000..a24e93ad9 --- /dev/null +++ b/rqueue-spring/src/main/java/com/github/sonus21/rqueue/spring/RqueueNatsListenerConfig.java @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ +package com.github.sonus21.rqueue.spring; + +import com.github.sonus21.rqueue.core.spi.MessageBroker; +import com.github.sonus21.rqueue.nats.RqueueNatsConfig; +import com.github.sonus21.rqueue.nats.internal.NatsProvisioner; +import com.github.sonus21.rqueue.nats.js.JetStreamMessageBroker; +import com.github.sonus21.rqueue.nats.js.NatsStreamValidator; +import io.nats.client.Connection; +import io.nats.client.JetStream; +import io.nats.client.JetStreamManagement; +import io.nats.client.Nats; +import io.nats.client.Options; +import java.io.IOException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.Environment; + +/** + * Non-Boot configuration that wires a JetStream-backed {@link MessageBroker} for Spring users + * who opt in via {@code @EnableRqueue(backend = NATS)} or via the {@link NatsBackendCondition} + * (jnats classpath + {@code rqueue.backend=nats}). + */ +@Configuration +public class RqueueNatsListenerConfig { + + @Autowired + Environment environment; + + @Bean + public Connection natsConnection() throws IOException { + String url = environment.getProperty("rqueue.nats.url", Options.DEFAULT_URL); + String username = environment.getProperty("rqueue.nats.username"); + String password = environment.getProperty("rqueue.nats.password"); + String token = environment.getProperty("rqueue.nats.token"); + String connectionName = environment.getProperty("rqueue.nats.connection-name"); + Options.Builder ob = new Options.Builder().server(url); + if (connectionName != null) { + ob.connectionName(connectionName); + } + if (token != null && !token.isEmpty()) { + ob.token(token.toCharArray()); + } else if (username != null && password != null) { + ob.userInfo(username, password); + } + try { + return Nats.connect(ob.build()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted while connecting to NATS", e); + } + } + + @Bean + public JetStream jetStream(Connection connection) throws IOException { + return connection.jetStream(); + } + + @Bean + public JetStreamManagement jetStreamManagement(Connection connection) throws IOException { + return connection.jetStreamManagement(); + } + + @Bean + public MessageBroker jetStreamMessageBroker( + Connection connection, JetStream jetStream, JetStreamManagement jetStreamManagement) { + return JetStreamMessageBroker.builder() + .connection(connection) + .jetStream(jetStream) + .management(jetStreamManagement) + .build(); + } + + @Bean + public NatsStreamValidator natsStreamValidator(NatsProvisioner natsProvisioner) { + return new NatsStreamValidator(natsProvisioner, RqueueNatsConfig.defaults()); + } +} diff --git a/rqueue-spring/src/main/java/com/github/sonus21/rqueue/spring/RqueueRedisConfigImportSelector.java b/rqueue-spring/src/main/java/com/github/sonus21/rqueue/spring/RqueueRedisConfigImportSelector.java new file mode 100644 index 000000000..783ecff5d --- /dev/null +++ b/rqueue-spring/src/main/java/com/github/sonus21/rqueue/spring/RqueueRedisConfigImportSelector.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ +package com.github.sonus21.rqueue.spring; + +import org.springframework.context.annotation.ImportSelector; +import org.springframework.core.type.AnnotationMetadata; +import org.springframework.util.ClassUtils; + +/** + * Conditionally imports {@code RqueueRedisListenerConfig} only when the {@code rqueue-redis} module + * is on the classpath. A direct {@code @Import(RqueueRedisListenerConfig.class)} would cause Spring + * to read the class bytecode before any {@code @Conditional} can fire, throwing + * {@code FileNotFoundException} when the module is excluded. + */ +public class RqueueRedisConfigImportSelector implements ImportSelector { + + private static final String REDIS_CONFIG_CLASS = + "com.github.sonus21.rqueue.redis.config.RqueueRedisListenerConfig"; + + @Override + public String[] selectImports(AnnotationMetadata importingClassMetadata) { + if (ClassUtils.isPresent(REDIS_CONFIG_CLASS, getClass().getClassLoader())) { + return new String[] {REDIS_CONFIG_CLASS}; + } + return new String[0]; + } +} diff --git a/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/RqueueNatsListenerConfigTest.java b/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/RqueueNatsListenerConfigTest.java new file mode 100644 index 000000000..345cca5f7 --- /dev/null +++ b/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/RqueueNatsListenerConfigTest.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ +package com.github.sonus21.rqueue.spring; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; + +import com.github.sonus21.rqueue.core.spi.MessageBroker; +import com.github.sonus21.rqueue.nats.js.JetStreamMessageBroker; +import io.nats.client.Connection; +import io.nats.client.JetStream; +import io.nats.client.JetStreamManagement; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Tag("unit") +@Tag("nats") +class RqueueNatsListenerConfigTest { + + @Configuration + static class MockBeans { + @Bean + Connection natsConnection() { + return mock(Connection.class); + } + + @Bean + JetStream jetStream() { + return mock(JetStream.class); + } + + @Bean + JetStreamManagement jetStreamManagement() { + return mock(JetStreamManagement.class); + } + + @Bean + MessageBroker jetStreamMessageBroker( + Connection connection, JetStream jetStream, JetStreamManagement management) { + return JetStreamMessageBroker.builder() + .connection(connection) + .jetStream(jetStream) + .management(management) + .build(); + } + } + + @Test + void natsBrokerBeanIsRegistered() { + AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(); + ctx.register(MockBeans.class); + ctx.refresh(); + MessageBroker broker = ctx.getBean(MessageBroker.class); + assertNotNull(broker); + assertTrue(broker instanceof JetStreamMessageBroker); + ctx.close(); + } +} diff --git a/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/unit/RqueueMessageConfigTest.java b/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/unit/RqueueMessageConfigTest.java index 60e943726..5e334ba92 100644 --- a/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/unit/RqueueMessageConfigTest.java +++ b/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/unit/RqueueMessageConfigTest.java @@ -18,15 +18,20 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.when; import com.github.sonus21.TestBase; import com.github.sonus21.rqueue.config.SimpleRqueueListenerContainerFactory; import com.github.sonus21.rqueue.converter.DefaultMessageConverterProvider; import com.github.sonus21.rqueue.converter.GenericMessageConverter; import com.github.sonus21.rqueue.core.DefaultRqueueMessageConverter; +import com.github.sonus21.rqueue.core.RqueueMessageEnqueuer; import com.github.sonus21.rqueue.core.RqueueMessageTemplate; import com.github.sonus21.rqueue.core.impl.UuidV4RqueueMessageIdGenerator; +import com.github.sonus21.rqueue.core.spi.Capabilities; +import com.github.sonus21.rqueue.core.spi.MessageBroker; import com.github.sonus21.rqueue.listener.RqueueMessageHandler; import com.github.sonus21.rqueue.spring.RqueueListenerConfig; import com.github.sonus21.rqueue.spring.tests.SpringUnitTest; @@ -50,6 +55,9 @@ class RqueueMessageConfigTest extends TestBase { @Mock RqueueMessageHandler rqueueMessageHandler; + @Mock + private MessageBroker messageBroker; + @Mock private SimpleRqueueListenerContainerFactory simpleRqueueListenerContainerFactory; @@ -79,7 +87,8 @@ void rqueueMessageHandlerDefaultCreation() throws IllegalAccessException { "messageConverterProviderClass", "com.github.sonus21.rqueue.converter.DefaultMessageConverterProvider", true); - assertNotNull(rqueueMessageConfig.rqueueMessageHandler()); + when(messageBroker.capabilities()).thenReturn(Capabilities.REDIS_DEFAULTS); + assertNotNull(rqueueMessageConfig.rqueueMessageHandler(messageBroker)); } @Test @@ -93,8 +102,10 @@ void rqueueMessageHandlerReused() throws IllegalAccessException { "com.github.sonus21.rqueue.converter.DefaultMessageConverterProvider", true); FieldUtils.writeField(messageConfig, "simpleRqueueListenerContainerFactory", factory, true); + when(messageBroker.capabilities()).thenReturn(Capabilities.REDIS_DEFAULTS); assertEquals( - rqueueMessageHandler.hashCode(), messageConfig.rqueueMessageHandler().hashCode()); + rqueueMessageHandler.hashCode(), + messageConfig.rqueueMessageHandler(messageBroker).hashCode()); } @Test @@ -114,14 +125,22 @@ void rqueueMessageListenerContainer() throws IllegalAccessException { } @Test - void rqueueMessageSenderWithMessageTemplate() throws IllegalAccessException { + void rqueueMessageEnqueuerWiresBroker() throws IllegalAccessException { SimpleRqueueListenerContainerFactory factory = new SimpleRqueueListenerContainerFactory(); factory.setRqueueMessageTemplate(rqueueMessageTemplate); doReturn(new DefaultRqueueMessageConverter()).when(rqueueMessageHandler).getMessageConverter(); RqueueListenerConfig messageConfig = new RqueueListenerConfig(); FieldUtils.writeField(messageConfig, "simpleRqueueListenerContainerFactory", factory, true); - assertNotNull(messageConfig.rqueueMessageEnqueuer( - rqueueMessageHandler, rqueueMessageTemplate, new UuidV4RqueueMessageIdGenerator())); - assertEquals(factory.getRqueueMessageTemplate().hashCode(), rqueueMessageTemplate.hashCode()); + + RqueueMessageEnqueuer enqueuer = messageConfig.rqueueMessageEnqueuer( + rqueueMessageHandler, + rqueueMessageTemplate, + messageBroker, + new UuidV4RqueueMessageIdGenerator()); + + assertNotNull(enqueuer); + // Broker is on the enqueuer (via BaseMessageSender), not on the template — sidesteps the + // Redis cycle and removes the original NPE class entirely. + assertSame(messageBroker, FieldUtils.readField(enqueuer, "messageBroker", true)); } } diff --git a/rqueue-test-util/build.gradle b/rqueue-test-util/build.gradle index 7ef6db9a1..b11d5f2f5 100644 --- a/rqueue-test-util/build.gradle +++ b/rqueue-test-util/build.gradle @@ -30,4 +30,13 @@ dependencies { api "org.hibernate.orm:hibernate-core:${hibernateCoreVersion}" api "com.athaydes.javanna:javanna:1.1" + + // Needed so shared test-utility classes (e.g. @CoreUnitTest) can reference Mockito's + // JUnit Jupiter extension from main sources. + api "org.mockito:mockito-junit-jupiter:${mockitoVersion}" + + // Test fixtures that ship from main sources (TestUtils, QueueStatisticsFixtures) reference + // model and service types from rqueue-core. Marked compileOnly to avoid a hard runtime + // coupling — every consumer of rqueue-test-util already has rqueue-core on its classpath. + compileOnly project(":rqueue-core") } diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/CoreUnitTest.java b/rqueue-test-util/src/main/java/com/github/sonus21/rqueue/CoreUnitTest.java similarity index 100% rename from rqueue-core/src/test/java/com/github/sonus21/rqueue/CoreUnitTest.java rename to rqueue-test-util/src/main/java/com/github/sonus21/rqueue/CoreUnitTest.java diff --git a/rqueue-test-util/src/main/java/com/github/sonus21/rqueue/models/db/QueueStatisticsFixtures.java b/rqueue-test-util/src/main/java/com/github/sonus21/rqueue/models/db/QueueStatisticsFixtures.java new file mode 100644 index 000000000..09cb9784e --- /dev/null +++ b/rqueue-test-util/src/main/java/com/github/sonus21/rqueue/models/db/QueueStatisticsFixtures.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2020-2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ + +package com.github.sonus21.rqueue.models.db; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.time.LocalDate; + +/** + * Shared fixture helpers for {@link QueueStatistics}. Lives in {@code rqueue-test-util/src/main} + * so it can be consumed by both {@code rqueue-core}'s tests and downstream backend modules + * ({@code rqueue-redis}, {@code rqueue-nats}) that exercise dashboard/chart code. + */ +public final class QueueStatisticsFixtures { + + private QueueStatisticsFixtures() {} + + public static void validate(QueueStatistics queueStatistics, int count) { + assertEquals(count, queueStatistics.getJobRunTime().size()); + assertEquals(count, queueStatistics.getTasksSuccessful().size()); + assertEquals(count, queueStatistics.getTasksDiscarded().size()); + assertEquals(count, queueStatistics.getTasksMovedToDeadLetter().size()); + assertEquals(count, queueStatistics.getTasksRetried().size()); + } + + public static void checkNonNull(QueueStatistics queueStatistics, String date) { + assertNotNull(queueStatistics.jobRunTime(date)); + assertTrue(queueStatistics.tasksSuccessful(date) > 0); + assertTrue(queueStatistics.tasksDiscarded(date) > 0); + assertTrue(queueStatistics.tasksMovedToDeadLetter(date) > 0); + assertTrue(queueStatistics.tasksRetried(date) > 0); + } + + public static void addData(QueueStatistics queueStatistics, LocalDate localDate, int day) { + String date = localDate.minusDays(day).toString(); + int val = 1 + (int) (Math.random() * 100); + queueStatistics.incrementSuccessful(date, val); + queueStatistics.incrementDiscard(date, val); + queueStatistics.incrementDeadLetter(date, val); + queueStatistics.incrementRetry(date, val); + int val2 = 1 + (int) (Math.random() * 100); + JobRunTime jobRunTime = new JobRunTime(val, val2, val * val2, val2); + queueStatistics.updateJobExecutionTime(date, jobRunTime); + } +} diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/utils/RqueueMessageTestUtils.java b/rqueue-test-util/src/main/java/com/github/sonus21/rqueue/utils/RqueueMessageTestUtils.java similarity index 100% rename from rqueue-core/src/test/java/com/github/sonus21/rqueue/utils/RqueueMessageTestUtils.java rename to rqueue-test-util/src/main/java/com/github/sonus21/rqueue/utils/RqueueMessageTestUtils.java diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/utils/TestUtils.java b/rqueue-test-util/src/main/java/com/github/sonus21/rqueue/utils/TestUtils.java similarity index 100% rename from rqueue-core/src/test/java/com/github/sonus21/rqueue/utils/TestUtils.java rename to rqueue-test-util/src/main/java/com/github/sonus21/rqueue/utils/TestUtils.java diff --git a/rqueue-web/build.gradle b/rqueue-web/build.gradle new file mode 100644 index 000000000..9105d1770 --- /dev/null +++ b/rqueue-web/build.gradle @@ -0,0 +1,57 @@ +plugins { + id 'com.vanniktech.maven.publish' version '0.28.0' +} +apply from: "${rootDir}/gradle/packaging.gradle" +apply from: "${rootDir}/gradle/test-runner.gradle" +apply from: "${rootDir}/gradle/code-publish.gradle" + +import com.vanniktech.maven.publish.SonatypeHost; + +mavenPublishing { + publishToMavenCentral(SonatypeHost.CENTRAL_PORTAL) + signAllPublications() + pom { + name = "Rqueue Web" + description = "Web dashboard, REST/reactive controllers, Pebble templates, and dashboard service interfaces for Rqueue. Pulled in by default through rqueue-spring-boot-starter; can be excluded for headless worker deployments that do not need the dashboard." + url = "https://github.com/sonus21/rqueue" + licenses { + license { + name = "Apache License 2.0" + url = "https://www.apache.org/licenses/LICENSE-2.0.txt" + } + } + developers { + developer { + id = "sonus21" + name = "Sonu Kumar" + email = "sonunitw12@gmail.com" + } + } + scm { + url = "https://github.com/sonus21/rqueue" + connection = "scm:git:git://github.com/sonus21/rqueue.git" + developerConnection = "scm:git:ssh://git@github.com:sonus21/rqueue.git" + } + issueManagement { + system = "GitHub" + url = "https://github.com/sonus21/rqueue/issues" + } + } +} + +dependencies { + api project(":rqueue-core") + api "jakarta.servlet:jakarta.servlet-api:${jakartaServletVersion}" + api "jakarta.validation:jakarta.validation-api:${jakartaValidationApiVersion}" + implementation 'jakarta.el:jakarta.el-api:5.0.0' + runtimeOnly 'org.glassfish:jakarta.el:4.0.2' + api "org.springframework:spring-webmvc:${springVersion}" + api "org.springframework:spring-webflux:${springVersion}" + api "io.pebbletemplates:pebble-spring7:${pebbleVersion}" + api "io.seruco.encoding:base62:${serucoEncodingVersion}" + api "org.hibernate.validator:hibernate-validator:${hibernateValidatorVersion}" + + testImplementation project(":rqueue-test-util") + testImplementation "io.lettuce:lettuce-core:${lettuceVersion}" + testImplementation "io.projectreactor:reactor-test:${projectReactorReactorTestVersion}" +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/pebble/DateTimeFunction.java b/rqueue-web/src/main/java/com/github/sonus21/rqueue/utils/pebble/DateTimeFunction.java similarity index 100% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/pebble/DateTimeFunction.java rename to rqueue-web/src/main/java/com/github/sonus21/rqueue/utils/pebble/DateTimeFunction.java diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/pebble/DeadLetterQueuesFunction.java b/rqueue-web/src/main/java/com/github/sonus21/rqueue/utils/pebble/DeadLetterQueuesFunction.java similarity index 100% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/pebble/DeadLetterQueuesFunction.java rename to rqueue-web/src/main/java/com/github/sonus21/rqueue/utils/pebble/DeadLetterQueuesFunction.java diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/pebble/DefaultFunction.java b/rqueue-web/src/main/java/com/github/sonus21/rqueue/utils/pebble/DefaultFunction.java similarity index 100% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/pebble/DefaultFunction.java rename to rqueue-web/src/main/java/com/github/sonus21/rqueue/utils/pebble/DefaultFunction.java diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/pebble/DurationFunction.java b/rqueue-web/src/main/java/com/github/sonus21/rqueue/utils/pebble/DurationFunction.java similarity index 100% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/pebble/DurationFunction.java rename to rqueue-web/src/main/java/com/github/sonus21/rqueue/utils/pebble/DurationFunction.java diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/pebble/ReadableDateTimeFunction.java b/rqueue-web/src/main/java/com/github/sonus21/rqueue/utils/pebble/ReadableDateTimeFunction.java similarity index 100% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/pebble/ReadableDateTimeFunction.java rename to rqueue-web/src/main/java/com/github/sonus21/rqueue/utils/pebble/ReadableDateTimeFunction.java diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/pebble/ResourceLoader.java b/rqueue-web/src/main/java/com/github/sonus21/rqueue/utils/pebble/ResourceLoader.java similarity index 100% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/pebble/ResourceLoader.java rename to rqueue-web/src/main/java/com/github/sonus21/rqueue/utils/pebble/ResourceLoader.java diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/pebble/RqueuePebbleExtension.java b/rqueue-web/src/main/java/com/github/sonus21/rqueue/utils/pebble/RqueuePebbleExtension.java similarity index 100% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/pebble/RqueuePebbleExtension.java rename to rqueue-web/src/main/java/com/github/sonus21/rqueue/utils/pebble/RqueuePebbleExtension.java diff --git a/rqueue-web/src/main/java/com/github/sonus21/rqueue/web/config/RqueueWebViewConfig.java b/rqueue-web/src/main/java/com/github/sonus21/rqueue/web/config/RqueueWebViewConfig.java new file mode 100644 index 000000000..6e1447a9c --- /dev/null +++ b/rqueue-web/src/main/java/com/github/sonus21/rqueue/web/config/RqueueWebViewConfig.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2020-2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ + +package com.github.sonus21.rqueue.web.config; + +import com.github.sonus21.rqueue.utils.condition.ReactiveEnabled; +import com.github.sonus21.rqueue.utils.pebble.ResourceLoader; +import com.github.sonus21.rqueue.utils.pebble.RqueuePebbleExtension; +import io.pebbletemplates.pebble.PebbleEngine; +import io.pebbletemplates.spring.extension.SpringExtension; +import io.pebbletemplates.spring.reactive.PebbleReactiveViewResolver; +import io.pebbletemplates.spring.servlet.PebbleViewResolver; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.servlet.ViewResolver; + +/** + * Wires Pebble template view resolvers for the rqueue dashboard. Lives in {@code rqueue-web} so + * headless deployments that exclude this module do not pull in Pebble or the Servlet/WebFlux view + * stack. Picked up automatically by the existing {@code com.github.sonus21.rqueue.web} component + * scan in {@code RqueueListenerConfig} / {@code RqueueListenerAutoConfig}. + */ +@Configuration +public class RqueueWebViewConfig { + + private static final String TEMPLATE_DIR = "templates/rqueue/"; + private static final String TEMPLATE_SUFFIX = ".html"; + + private PebbleEngine createPebbleEngine() { + ResourceLoader loader = new ResourceLoader(); + loader.setPrefix(TEMPLATE_DIR); + loader.setSuffix(TEMPLATE_SUFFIX); + return new PebbleEngine.Builder() + .extension(new RqueuePebbleExtension(), new SpringExtension(null)) + .loader(loader) + .build(); + } + + @Bean + public ViewResolver rqueueViewResolver() { + PebbleViewResolver resolver = new PebbleViewResolver(createPebbleEngine()); + resolver.setPrefix(TEMPLATE_DIR); + resolver.setSuffix(TEMPLATE_SUFFIX); + return resolver; + } + + @Bean + @Conditional(ReactiveEnabled.class) + public org.springframework.web.reactive.result.view.ViewResolver reactiveRqueueViewResolver() { + PebbleReactiveViewResolver resolver = new PebbleReactiveViewResolver(createPebbleEngine()); + resolver.setPrefix(TEMPLATE_DIR); + resolver.setSuffix(TEMPLATE_SUFFIX); + return resolver; + } +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/BaseController.java b/rqueue-web/src/main/java/com/github/sonus21/rqueue/web/controller/BaseController.java similarity index 100% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/BaseController.java rename to rqueue-web/src/main/java/com/github/sonus21/rqueue/web/controller/BaseController.java diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/BaseReactiveController.java b/rqueue-web/src/main/java/com/github/sonus21/rqueue/web/controller/BaseReactiveController.java similarity index 100% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/BaseReactiveController.java rename to rqueue-web/src/main/java/com/github/sonus21/rqueue/web/controller/BaseReactiveController.java diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/ReactiveRqueueRestController.java b/rqueue-web/src/main/java/com/github/sonus21/rqueue/web/controller/ReactiveRqueueRestController.java similarity index 88% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/ReactiveRqueueRestController.java rename to rqueue-web/src/main/java/com/github/sonus21/rqueue/web/controller/ReactiveRqueueRestController.java index 23744b74e..bbee5693d 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/ReactiveRqueueRestController.java +++ b/rqueue-web/src/main/java/com/github/sonus21/rqueue/web/controller/ReactiveRqueueRestController.java @@ -35,12 +35,12 @@ import com.github.sonus21.rqueue.models.response.DataViewResponse; import com.github.sonus21.rqueue.models.response.MessageMoveResponse; import com.github.sonus21.rqueue.models.response.StringResponse; +import com.github.sonus21.rqueue.service.RqueueUtilityService; import com.github.sonus21.rqueue.utils.condition.ReactiveEnabled; import com.github.sonus21.rqueue.web.service.RqueueDashboardChartService; import com.github.sonus21.rqueue.web.service.RqueueJobService; import com.github.sonus21.rqueue.web.service.RqueueQDetailService; import com.github.sonus21.rqueue.web.service.RqueueSystemManagerService; -import com.github.sonus21.rqueue.web.service.RqueueUtilityService; import jakarta.validation.Valid; import jakarta.validation.constraints.NotEmpty; import org.springframework.beans.factory.annotation.Autowired; @@ -55,8 +55,8 @@ import org.springframework.web.bind.annotation.RestController; import reactor.core.publisher.Mono; -@RestController @Conditional(ReactiveEnabled.class) +@RestController @RequestMapping(path = "${rqueue.web.url.prefix:}rqueue/api/v1") public class ReactiveRqueueRestController extends BaseReactiveController { @@ -65,6 +65,9 @@ public class ReactiveRqueueRestController extends BaseReactiveController { private final RqueueUtilityService rqueueUtilityService; private final RqueueSystemManagerService rqueueQManagerService; private final RqueueJobService rqueueJobService; + private final org.springframework.beans.factory.ObjectProvider< + com.github.sonus21.rqueue.core.spi.MessageBroker> + messageBrokerProvider; @Autowired public ReactiveRqueueRestController( @@ -73,13 +76,17 @@ public ReactiveRqueueRestController( RqueueUtilityService rqueueUtilityService, RqueueSystemManagerService rqueueQManagerService, RqueueWebConfig rqueueWebConfig, - RqueueJobService rqueueJobService) { + RqueueJobService rqueueJobService, + org.springframework.beans.factory.ObjectProvider< + com.github.sonus21.rqueue.core.spi.MessageBroker> + messageBrokerProvider) { super(rqueueWebConfig); this.rqueueDashboardChartService = rqueueDashboardChartService; this.rqueueQDetailService = rqueueQDetailService; this.rqueueUtilityService = rqueueUtilityService; this.rqueueQManagerService = rqueueQManagerService; this.rqueueJobService = rqueueJobService; + this.messageBrokerProvider = messageBrokerProvider; } @PostMapping("chart") @@ -215,4 +222,20 @@ public Mono aggregateDataCounter( } return null; } + + /** Reactive twin of {@code RqueueRestController#capabilities}. See javadoc there. */ + @GetMapping("capabilities") + @ResponseBody + public Mono capabilities( + ServerHttpResponse response) { + if (!isEnabled(response)) { + return Mono.empty(); + } + com.github.sonus21.rqueue.core.spi.MessageBroker broker = + messageBrokerProvider.getIfAvailable(); + return Mono.just( + broker == null + ? com.github.sonus21.rqueue.core.spi.Capabilities.REDIS_DEFAULTS + : broker.capabilities()); + } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/ReactiveRqueueViewController.java b/rqueue-web/src/main/java/com/github/sonus21/rqueue/web/controller/ReactiveRqueueViewController.java similarity index 100% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/ReactiveRqueueViewController.java rename to rqueue-web/src/main/java/com/github/sonus21/rqueue/web/controller/ReactiveRqueueViewController.java index c22c36927..858003af4 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/ReactiveRqueueViewController.java +++ b/rqueue-web/src/main/java/com/github/sonus21/rqueue/web/controller/ReactiveRqueueViewController.java @@ -35,9 +35,9 @@ import org.springframework.web.reactive.result.view.ViewResolver; import reactor.core.publisher.Mono; +@Conditional(ReactiveEnabled.class) @Controller @RequestMapping(path = "${rqueue.web.url.prefix:}rqueue") -@Conditional(ReactiveEnabled.class) public class ReactiveRqueueViewController extends BaseReactiveController { private final ViewResolver rqueueViewResolver; diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/RqueueRestController.java b/rqueue-web/src/main/java/com/github/sonus21/rqueue/web/controller/RqueueRestController.java similarity index 84% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/RqueueRestController.java rename to rqueue-web/src/main/java/com/github/sonus21/rqueue/web/controller/RqueueRestController.java index b9f8a4e00..e55ad3d79 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/RqueueRestController.java +++ b/rqueue-web/src/main/java/com/github/sonus21/rqueue/web/controller/RqueueRestController.java @@ -17,6 +17,8 @@ package com.github.sonus21.rqueue.web.controller; import com.github.sonus21.rqueue.config.RqueueWebConfig; +import com.github.sonus21.rqueue.core.spi.Capabilities; +import com.github.sonus21.rqueue.core.spi.MessageBroker; import com.github.sonus21.rqueue.exception.ProcessingException; import com.github.sonus21.rqueue.models.enums.AggregationType; import com.github.sonus21.rqueue.models.request.ChartDataRequest; @@ -35,15 +37,16 @@ import com.github.sonus21.rqueue.models.response.DataViewResponse; import com.github.sonus21.rqueue.models.response.MessageMoveResponse; import com.github.sonus21.rqueue.models.response.StringResponse; +import com.github.sonus21.rqueue.service.RqueueUtilityService; import com.github.sonus21.rqueue.utils.condition.ReactiveDisabled; import com.github.sonus21.rqueue.web.service.RqueueDashboardChartService; import com.github.sonus21.rqueue.web.service.RqueueJobService; import com.github.sonus21.rqueue.web.service.RqueueQDetailService; import com.github.sonus21.rqueue.web.service.RqueueSystemManagerService; -import com.github.sonus21.rqueue.web.service.RqueueUtilityService; import jakarta.servlet.http.HttpServletResponse; import jakarta.validation.Valid; import jakarta.validation.constraints.NotEmpty; +import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Conditional; import org.springframework.web.bind.annotation.GetMapping; @@ -54,9 +57,9 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; +@Conditional(ReactiveDisabled.class) @RestController @RequestMapping(path = "${rqueue.web.url.prefix:}rqueue/api/v1") -@Conditional(ReactiveDisabled.class) public class RqueueRestController extends BaseController { private final RqueueDashboardChartService rqueueDashboardChartService; @@ -64,6 +67,7 @@ public class RqueueRestController extends BaseController { private final RqueueUtilityService rqueueUtilityService; private final RqueueSystemManagerService rqueueQManagerService; private final RqueueJobService rqueueJobService; + private final ObjectProvider messageBrokerProvider; @Autowired public RqueueRestController( @@ -72,13 +76,15 @@ public RqueueRestController( RqueueUtilityService rqueueUtilityService, RqueueSystemManagerService rqueueQManagerService, RqueueWebConfig rqueueWebConfig, - RqueueJobService rqueueJobService) { + RqueueJobService rqueueJobService, + ObjectProvider messageBrokerProvider) { super(rqueueWebConfig); this.rqueueDashboardChartService = rqueueDashboardChartService; this.rqueueQDetailService = rqueueQDetailService; this.rqueueUtilityService = rqueueUtilityService; this.rqueueQManagerService = rqueueQManagerService; this.rqueueJobService = rqueueJobService; + this.messageBrokerProvider = messageBrokerProvider; } @PostMapping("chart") @@ -212,4 +218,25 @@ public DataSelectorResponse aggregateDataCounter( } return null; } + + /** + * Reports the active broker's capability flags (delayed enqueue, scheduled introspection, + * cron jobs, primary-handler dispatch). The dashboard front-end reads this once at boot to + * hide panels the backend cannot service rather than relying on per-call 501 responses. + * + *

Sourced from {@link MessageBroker#capabilities()} — the Redis broker answers with + * {@link Capabilities#REDIS_DEFAULTS} (everything true) and the NATS broker answers + * with everything false in v1. {@link ObjectProvider} keeps the broker dependency optional + * so deployments that strip the bean still get a sane payload (defaults to Redis, the + * historical behavior). + */ + @GetMapping("capabilities") + @ResponseBody + public Capabilities capabilities(HttpServletResponse response) { + if (!isEnable(response)) { + return null; + } + MessageBroker broker = messageBrokerProvider.getIfAvailable(); + return broker == null ? Capabilities.REDIS_DEFAULTS : broker.capabilities(); + } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/RqueueViewController.java b/rqueue-web/src/main/java/com/github/sonus21/rqueue/web/controller/RqueueViewController.java similarity index 100% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/RqueueViewController.java rename to rqueue-web/src/main/java/com/github/sonus21/rqueue/web/controller/RqueueViewController.java index 6e4dafd1f..fc7a39df0 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/controller/RqueueViewController.java +++ b/rqueue-web/src/main/java/com/github/sonus21/rqueue/web/controller/RqueueViewController.java @@ -34,9 +34,9 @@ import org.springframework.web.servlet.View; import org.springframework.web.servlet.ViewResolver; +@Conditional(ReactiveDisabled.class) @Controller @RequestMapping(path = "${rqueue.web.url.prefix:}rqueue") -@Conditional(ReactiveDisabled.class) public class RqueueViewController extends BaseController { private final ViewResolver rqueueViewResolver; diff --git a/rqueue-web/src/main/java/com/github/sonus21/rqueue/web/controller/RqueueWebExceptionAdvice.java b/rqueue-web/src/main/java/com/github/sonus21/rqueue/web/controller/RqueueWebExceptionAdvice.java new file mode 100644 index 000000000..9712bf43e --- /dev/null +++ b/rqueue-web/src/main/java/com/github/sonus21/rqueue/web/controller/RqueueWebExceptionAdvice.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2026 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + * + */ + +package com.github.sonus21.rqueue.web.controller; + +import com.github.sonus21.rqueue.exception.BackendCapabilityException; +import java.util.LinkedHashMap; +import java.util.Map; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.RestControllerAdvice; + +/** + * Maps rqueue web exceptions to HTTP responses. Scoped to rqueue's controller package so it does + * not interfere with the host application's exception handling. + */ +@RestControllerAdvice(basePackageClasses = RqueueWebExceptionAdvice.class) +public class RqueueWebExceptionAdvice { + + @ExceptionHandler(BackendCapabilityException.class) + public ResponseEntity> handleBackendCapability( + BackendCapabilityException ex) { + Map body = new LinkedHashMap<>(); + body.put("code", HttpStatus.NOT_IMPLEMENTED.value()); + body.put("backend", ex.getBackend()); + body.put("operation", ex.getOperation()); + body.put("message", ex.getMessage()); + return ResponseEntity.status(HttpStatus.NOT_IMPLEMENTED).body(body); + } +} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueDashboardChartService.java b/rqueue-web/src/main/java/com/github/sonus21/rqueue/web/service/RqueueDashboardChartService.java similarity index 100% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueDashboardChartService.java rename to rqueue-web/src/main/java/com/github/sonus21/rqueue/web/service/RqueueDashboardChartService.java diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueJobMetricsAggregatorService.java b/rqueue-web/src/main/java/com/github/sonus21/rqueue/web/service/RqueueJobMetricsAggregatorService.java similarity index 100% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueJobMetricsAggregatorService.java rename to rqueue-web/src/main/java/com/github/sonus21/rqueue/web/service/RqueueJobMetricsAggregatorService.java diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueJobService.java b/rqueue-web/src/main/java/com/github/sonus21/rqueue/web/service/RqueueJobService.java similarity index 100% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueJobService.java rename to rqueue-web/src/main/java/com/github/sonus21/rqueue/web/service/RqueueJobService.java diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueQDetailService.java b/rqueue-web/src/main/java/com/github/sonus21/rqueue/web/service/RqueueQDetailService.java similarity index 97% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueQDetailService.java rename to rqueue-web/src/main/java/com/github/sonus21/rqueue/web/service/RqueueQDetailService.java index 3153dbefa..ccd16c269 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueQDetailService.java +++ b/rqueue-web/src/main/java/com/github/sonus21/rqueue/web/service/RqueueQDetailService.java @@ -29,6 +29,10 @@ public interface RqueueQDetailService { + String storageKicker(); + + String storageDescription(); + Map>> getQueueDataStructureDetails( List queueConfig); diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueSystemManagerService.java b/rqueue-web/src/main/java/com/github/sonus21/rqueue/web/service/RqueueSystemManagerService.java similarity index 100% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueSystemManagerService.java rename to rqueue-web/src/main/java/com/github/sonus21/rqueue/web/service/RqueueSystemManagerService.java diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueViewControllerService.java b/rqueue-web/src/main/java/com/github/sonus21/rqueue/web/service/RqueueViewControllerService.java similarity index 100% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/RqueueViewControllerService.java rename to rqueue-web/src/main/java/com/github/sonus21/rqueue/web/service/RqueueViewControllerService.java diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueDashboardChartServiceImpl.java b/rqueue-web/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueDashboardChartServiceImpl.java similarity index 100% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueDashboardChartServiceImpl.java rename to rqueue-web/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueDashboardChartServiceImpl.java diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueJobServiceImpl.java b/rqueue-web/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueJobServiceImpl.java similarity index 91% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueJobServiceImpl.java rename to rqueue-web/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueJobServiceImpl.java index a1827e2f1..0a9f16b35 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueJobServiceImpl.java +++ b/rqueue-web/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueJobServiceImpl.java @@ -23,11 +23,12 @@ import com.github.sonus21.rqueue.models.response.DataViewResponse; import com.github.sonus21.rqueue.models.response.TableColumn; import com.github.sonus21.rqueue.models.response.TableRow; +import com.github.sonus21.rqueue.serdes.RqueueSerDes; import com.github.sonus21.rqueue.utils.Constants; import com.github.sonus21.rqueue.utils.DateTimeUtils; -import com.github.sonus21.rqueue.utils.SerializationUtils; import com.github.sonus21.rqueue.utils.StringUtils; import com.github.sonus21.rqueue.web.service.RqueueJobService; +import java.io.IOException; import java.util.Arrays; import java.util.LinkedList; import java.util.List; @@ -36,18 +37,17 @@ import org.springframework.util.CollectionUtils; import reactor.core.publisher.Mono; import tools.jackson.core.JacksonException; -import tools.jackson.databind.ObjectMapper; @Service public class RqueueJobServiceImpl implements RqueueJobService { private final RqueueJobDao rqueueJobDao; - private final ObjectMapper objectMapper; + private final RqueueSerDes rqueueSerDes; @Autowired - public RqueueJobServiceImpl(RqueueJobDao rqueueJobDao) { + public RqueueJobServiceImpl(RqueueJobDao rqueueJobDao, RqueueSerDes rqueueSerDes) { this.rqueueJobDao = rqueueJobDao; - this.objectMapper = SerializationUtils.createObjectMapper(); + this.rqueueSerDes = rqueueSerDes; } private TableRow getTableRow(RqueueJob job) throws ProcessingException { @@ -75,9 +75,9 @@ private TableRow getTableRow(RqueueJob job) throws ProcessingException { columns.add(new TableColumn(Constants.BLANK)); } else { try { - String data = objectMapper.writeValueAsString(job.getCheckins()); + String data = rqueueSerDes.serializeAsString(job.getCheckins()); columns.add(new TableColumn(data)); - } catch (JacksonException e) { + } catch (JacksonException | IOException e) { throw new ProcessingException(e.getMessage(), e); } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueQDetailServiceImpl.java b/rqueue-web/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueQDetailServiceImpl.java similarity index 62% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueQDetailServiceImpl.java rename to rqueue-web/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueQDetailServiceImpl.java index be1e044d8..a118f6dd1 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueQDetailServiceImpl.java +++ b/rqueue-web/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueQDetailServiceImpl.java @@ -19,12 +19,14 @@ import static com.github.sonus21.rqueue.utils.StringUtils.clean; import static com.google.common.collect.Lists.newArrayList; -import com.github.sonus21.rqueue.common.RqueueRedisTemplate; import com.github.sonus21.rqueue.config.RqueueConfig; +import com.github.sonus21.rqueue.core.EndpointRegistry; import com.github.sonus21.rqueue.core.RqueueMessage; import com.github.sonus21.rqueue.core.RqueueMessageTemplate; +import com.github.sonus21.rqueue.core.spi.MessageBroker; import com.github.sonus21.rqueue.core.support.RqueueMessageUtils; import com.github.sonus21.rqueue.exception.UnknownSwitchCase; +import com.github.sonus21.rqueue.listener.QueueDetail; import com.github.sonus21.rqueue.models.db.DeadLetterQueue; import com.github.sonus21.rqueue.models.db.MessageMetadata; import com.github.sonus21.rqueue.models.db.QueueConfig; @@ -40,11 +42,11 @@ import com.github.sonus21.rqueue.models.response.RowColumnMetaType; import com.github.sonus21.rqueue.models.response.TableColumn; import com.github.sonus21.rqueue.models.response.TableRow; +import com.github.sonus21.rqueue.repository.MessageBrowsingRepository; +import com.github.sonus21.rqueue.service.RqueueMessageMetadataService; import com.github.sonus21.rqueue.utils.Constants; import com.github.sonus21.rqueue.utils.DateTimeUtils; -import com.github.sonus21.rqueue.utils.RedisUtils; import com.github.sonus21.rqueue.utils.StringUtils; -import com.github.sonus21.rqueue.web.service.RqueueMessageMetadataService; import com.github.sonus21.rqueue.web.service.RqueueQDetailService; import com.github.sonus21.rqueue.web.service.RqueueSystemManagerService; import com.github.sonus21.rqueue.worker.RqueueWorkerRegistry; @@ -58,7 +60,6 @@ import java.util.Objects; import java.util.stream.Collectors; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.redis.core.DefaultTypedTuple; import org.springframework.data.redis.core.ZSetOperations.TypedTuple; import org.springframework.stereotype.Service; @@ -68,22 +69,31 @@ @Service public class RqueueQDetailServiceImpl implements RqueueQDetailService { - private final RqueueRedisTemplate stringRqueueRedisTemplate; + private final MessageBrowsingRepository messageBrowsingRepository; private final RqueueMessageTemplate rqueueMessageTemplate; private final RqueueSystemManagerService rqueueSystemManagerService; private final RqueueMessageMetadataService rqueueMessageMetadataService; private final RqueueConfig rqueueConfig; private final RqueueWorkerRegistry rqueueWorkerRegistry; + /** + * Optional broker SPI. When set (non-Redis backend), the dashboard prefers + * {@link MessageBroker#size(QueueDetail)} and + * {@link MessageBroker#peek(QueueDetail, long, long)} for read paths instead of + * the Redis DAOs. {@code @Autowired(required = false)} keeps the Redis-only path + * unchanged when no broker is configured. + */ + private MessageBroker messageBroker; + @Autowired public RqueueQDetailServiceImpl( - @Qualifier("stringRqueueRedisTemplate") RqueueRedisTemplate stringRqueueRedisTemplate, + MessageBrowsingRepository messageBrowsingRepository, RqueueMessageTemplate rqueueMessageTemplate, RqueueSystemManagerService rqueueSystemManagerService, RqueueMessageMetadataService rqueueMessageMetadataService, RqueueConfig rqueueConfig, RqueueWorkerRegistry rqueueWorkerRegistry) { - this.stringRqueueRedisTemplate = stringRqueueRedisTemplate; + this.messageBrowsingRepository = messageBrowsingRepository; this.rqueueMessageTemplate = rqueueMessageTemplate; this.rqueueSystemManagerService = rqueueSystemManagerService; this.rqueueMessageMetadataService = rqueueMessageMetadataService; @@ -91,6 +101,60 @@ public RqueueQDetailServiceImpl( this.rqueueWorkerRegistry = rqueueWorkerRegistry; } + @Autowired(required = false) + public void setMessageBroker(MessageBroker messageBroker) { + this.messageBroker = messageBroker; + } + + /** + * Visible for tests and pluggable backends. + */ + public MessageBroker getMessageBroker() { + return messageBroker; + } + + /** + * Resolves a {@link QueueDetail} for the given queue name. Returns {@code null} when no + * detail is registered (e.g. shutdown / late init); callers should fall back to the Redis + * path in that case. + */ + private QueueDetail lookupQueueDetail(String queueName) { + try { + return EndpointRegistry.get(queueName); + } catch (Exception e) { + return null; + } + } + + private boolean brokerHidesScheduled() { + return messageBroker != null && !messageBroker.capabilities().supportsScheduledIntrospection(); + } + + private boolean brokerHidesCron() { + return messageBroker != null && !messageBroker.capabilities().supportsCronJobs(); + } + + /** + * Returns true when the broker manages its own in-flight tracking and does not use the Redis + * processing ZSET. Brokers that return {@code usesPrimaryHandlerDispatch() == false} (e.g. NATS) + * have no separate "running" queue to inspect, so the RUNNING tab/row must be suppressed. + */ + private boolean brokerHidesRunning() { + return messageBroker != null && !messageBroker.capabilities().usesPrimaryHandlerDispatch(); + } + + @Override + public String storageKicker() { + return messageBroker != null ? messageBroker.storageKicker() : "Redis"; + } + + @Override + public String storageDescription() { + return messageBroker != null + ? messageBroker.storageDescription() + : "Underlying Redis structures for the queues visible on this page."; + } + @Override public Map>> getQueueDataStructureDetails( List queueConfig) { @@ -103,46 +167,73 @@ public List> getQueueDataStructureDetail(QueueCon if (queueConfig == null) { return Collections.emptyList(); } - Long pending = stringRqueueRedisTemplate.getListSize(queueConfig.getQueueName()); - String processingQueueName = queueConfig.getProcessingQueueName(); - Long running = stringRqueueRedisTemplate.getZsetSize(processingQueueName); - List> queueRedisDataDetails = newArrayList( - new HashMap.SimpleEntry<>( + // Route size lookup through the broker SPI when configured (non-Redis backend). + QueueDetail brokerQueueDetail = + messageBroker != null ? lookupQueueDetail(queueConfig.getName()) : null; + Long pending; + if (brokerQueueDetail != null) { + pending = messageBroker.size(brokerQueueDetail); + } else { + pending = messageBrowsingRepository.getDataSize(queueConfig.getQueueName(), DataType.LIST); + } + // When a non-Redis broker is configured, use its storage display names instead of Redis keys. + String pendingDisplayName = + brokerQueueDetail != null && messageBroker.storageDisplayName(brokerQueueDetail) != null + ? messageBroker.storageDisplayName(brokerQueueDetail) + : queueConfig.getQueueName(); + List> queueRedisDataDetails = + newArrayList(new HashMap.SimpleEntry<>( NavTab.PENDING, - new RedisDataDetail( - queueConfig.getQueueName(), DataType.LIST, pending == null ? 0 : pending)), - new HashMap.SimpleEntry<>( - NavTab.RUNNING, - new RedisDataDetail( - processingQueueName, DataType.ZSET, running == null ? 0 : running))); + new RedisDataDetail(pendingDisplayName, DataType.LIST, pending == null ? 0 : pending))); + // Brokers that manage their own in-flight tracking (e.g. NATS JetStream) have no separate + // processing ZSET, so omit the RUNNING entry to avoid a 501 when the explorer opens it. + if (!brokerHidesRunning()) { + String processingQueueName = queueConfig.getProcessingQueueName(); + Long running = messageBrowsingRepository.getDataSize(processingQueueName, DataType.ZSET); + queueRedisDataDetails.add(new HashMap.SimpleEntry<>( + NavTab.RUNNING, + new RedisDataDetail(processingQueueName, DataType.ZSET, running == null ? 0 : running))); + } String scheduledQueueName = queueConfig.getScheduledQueueName(); - Long scheduled = stringRqueueRedisTemplate.getZsetSize(scheduledQueueName); - queueRedisDataDetails.add(new HashMap.SimpleEntry<>( - NavTab.SCHEDULED, - new RedisDataDetail(scheduledQueueName, DataType.ZSET, scheduled == null ? 0 : scheduled))); + // When the broker doesn't support scheduled introspection (e.g. JetStream), suppress + // the SCHEDULED nav tab entry entirely so the dashboard doesn't query an absent ZSET. + if (!brokerHidesScheduled()) { + Long scheduled = messageBrowsingRepository.getDataSize(scheduledQueueName, DataType.ZSET); + queueRedisDataDetails.add(new HashMap.SimpleEntry<>( + NavTab.SCHEDULED, + new RedisDataDetail( + scheduledQueueName, DataType.ZSET, scheduled == null ? 0 : scheduled))); + } if (!CollectionUtils.isEmpty(queueConfig.getDeadLetterQueues())) { for (DeadLetterQueue dlq : queueConfig.getDeadLetterQueues()) { + String dlqDisplayName = brokerQueueDetail != null + && messageBroker.dlqStorageDisplayName(brokerQueueDetail) != null + ? messageBroker.dlqStorageDisplayName(brokerQueueDetail) + : dlq.getName(); if (!dlq.isConsumerEnabled()) { - Long dlqSize = stringRqueueRedisTemplate.getListSize(dlq.getName()); + Long dlqSize = messageBrowsingRepository.getDataSize(dlq.getName(), DataType.LIST); queueRedisDataDetails.add(new HashMap.SimpleEntry<>( NavTab.DEAD, - new RedisDataDetail(dlq.getName(), DataType.LIST, dlqSize == null ? 0 : dlqSize))); + new RedisDataDetail(dlqDisplayName, DataType.LIST, dlqSize == null ? 0 : dlqSize))); } else { // TODO should we redirect to the queue page? queueRedisDataDetails.add(new HashMap.SimpleEntry<>( - NavTab.DEAD, new RedisDataDetail(dlq.getName(), DataType.LIST, -1))); + NavTab.DEAD, new RedisDataDetail(dlqDisplayName, DataType.LIST, -1))); } } } if (rqueueConfig.messageInTerminalStateShouldBeStored() && !StringUtils.isEmpty(queueConfig.getCompletedQueueName())) { - Long completed = stringRqueueRedisTemplate.getZsetSize(queueConfig.getCompletedQueueName()); + Long completed = + messageBrowsingRepository.getDataSize(queueConfig.getCompletedQueueName(), DataType.ZSET); + String completedDisplayName = + brokerQueueDetail != null && messageBroker.storageDisplayName(brokerQueueDetail) != null + ? messageBroker.storageDisplayName(brokerQueueDetail) + : queueConfig.getCompletedQueueName(); queueRedisDataDetails.add(new HashMap.SimpleEntry<>( NavTab.COMPLETED, new RedisDataDetail( - queueConfig.getCompletedQueueName(), - DataType.ZSET, - completed == null ? 0 : completed))); + completedDisplayName, DataType.ZSET, completed == null ? 0 : completed))); } return queueRedisDataDetails; } @@ -152,8 +243,14 @@ public List getNavTabs(QueueConfig queueConfig) { List navTabs = new ArrayList<>(); if (queueConfig != null) { navTabs.add(NavTab.PENDING); - navTabs.add(NavTab.SCHEDULED); - navTabs.add(NavTab.RUNNING); + // Hide SCHEDULED tab for brokers without scheduled-queue introspection support. + if (!brokerHidesScheduled()) { + navTabs.add(NavTab.SCHEDULED); + } + // Hide RUNNING tab for brokers that manage in-flight tracking internally (e.g. NATS). + if (!brokerHidesRunning()) { + navTabs.add(NavTab.RUNNING); + } if (queueConfig.hasDeadLetterQueue()) { navTabs.add(NavTab.DEAD); } @@ -163,6 +260,7 @@ public List getNavTabs(QueueConfig queueConfig) { private List> readFromZset( String name, int pageNumber, int itemPerPage) { + requireScheduledIntrospection("readFromZset"); long start = pageNumber * (long) itemPerPage; long end = start + itemPerPage - 1; @@ -182,11 +280,27 @@ private List> readFromList( private List> readFromZetWithScore( String name, int pageNumber, int itemPerPage) { + requireScheduledIntrospection("readFromZsetWithScore"); long start = pageNumber * (long) itemPerPage; long end = start + itemPerPage - 1; return rqueueMessageTemplate.readFromZsetWithScore(name, start, end); } + /** + * Guard for ZSET-shaped lookups that the redis backend services natively but no other backend + * does. Backends that report {@code !supportsScheduledIntrospection()} surface a structured 501 + * via {@code BackendCapabilityException} instead of NPE-ing through a Redis-shaped template + * with no Redis connection. + */ + private void requireScheduledIntrospection(String op) { + if (messageBroker != null && !messageBroker.capabilities().supportsScheduledIntrospection()) { + throw new com.github.sonus21.rqueue.exception.BackendCapabilityException( + messageBroker.getClass().getSimpleName(), + op, + "broker does not expose scheduled / completion ZSET introspection"); + } + } + private List buildRows( List> rqueueMessages, RowBuilder rowBuilder) { if (CollectionUtils.isEmpty(rqueueMessages)) { @@ -244,9 +358,32 @@ public DataViewResponse getExplorePageData( boolean deadLetterQueue = queueConfig.isDeadLetterQueue(name); boolean scheduledQueue = queueConfig.getScheduledQueueName().equals(name); boolean completionQueue = name.equals(queueConfig.getCompletedQueueName()); + // Surface broker capability hints so the UI can hide the corresponding panels. + response.setHideScheduledPanel(brokerHidesScheduled()); + response.setHideCronJobs(brokerHidesCron()); + // When the broker does not support scheduled-queue introspection, return an empty + // result set for the scheduled tab. The hideScheduledPanel flag (above) tells the + // frontend to grey out / hide the panel. + if (scheduledQueue && brokerHidesScheduled()) { + response.setRows(Collections.emptyList()); + return response; + } setHeadersIfRequired(deadLetterQueue, completionQueue, type, response, pageNumber); addActionsIfRequired( src, name, type, scheduledQueue, deadLetterQueue, completionQueue, response); + // Prefer broker.peek() for the ready (LIST) queue when a non-Redis broker is configured. + if (type == DataType.LIST && !deadLetterQueue && messageBroker != null) { + QueueDetail qd = lookupQueueDetail(queueConfig.getName()); + if (qd != null) { + long offset = (long) pageNumber * itemPerPage; + List peeked = messageBroker.peek(qd, offset, itemPerPage); + List> tuples = peeked.stream() + .map(m -> (TypedTuple) new DefaultTypedTuple<>(m, null)) + .collect(Collectors.toList()); + response.setRows(buildRows(tuples, new ListRowBuilder(false))); + return response; + } + } switch (type) { case ZSET: if (scheduledQueue) { @@ -288,60 +425,6 @@ private List> readFromMessageMetadataStore( .collect(Collectors.toList()); } - private DataViewResponse responseForSet(String name) { - List items = new ArrayList<>(stringRqueueRedisTemplate.getMembers(name)); - DataViewResponse response = new DataViewResponse(); - response.setHeaders(Collections.singletonList("Item")); - List tableRows = new ArrayList<>(); - for (Object item : items) { - tableRows.add(new TableRow(new TableColumn(item.toString()))); - } - response.setRows(tableRows); - return response; - } - - private DataViewResponse responseForKeyVal(String name) { - DataViewResponse response = new DataViewResponse(); - response.setHeaders(Collections.singletonList("Value")); - Object val = stringRqueueRedisTemplate.get(name); - response.addRow(new TableRow(new TableColumn(String.valueOf(val)))); - return response; - } - - private DataViewResponse responseForZset( - String name, String key, int pageNumber, int itemPerPage) { - DataViewResponse response = new DataViewResponse(); - int start = pageNumber * itemPerPage; - int end = start + itemPerPage - 1; - List tableRows = new ArrayList<>(); - if (!StringUtils.isEmpty(key)) { - Double score = stringRqueueRedisTemplate.getZsetMemberScore(name, key); - response.setHeaders(Collections.singletonList("Score")); - tableRows.add(new TableRow(new TableColumn(score))); - } else { - response.setHeaders(Arrays.asList("Value", "Score")); - for (TypedTuple tuple : stringRqueueRedisTemplate.zrangeWithScore(name, start, end)) { - tableRows.add(new TableRow(Arrays.asList( - new TableColumn(String.valueOf(tuple.getValue())), new TableColumn(tuple.getScore())))); - } - } - response.setRows(tableRows); - return response; - } - - private DataViewResponse responseForList(String name, int pageNumber, int itemPerPage) { - DataViewResponse response = new DataViewResponse(); - response.setHeaders(Collections.singletonList("Item")); - int start = pageNumber * itemPerPage; - int end = start + itemPerPage - 1; - List tableRows = new ArrayList<>(); - for (Object s : stringRqueueRedisTemplate.lrange(name, start, end)) { - tableRows.add(new TableRow(new TableColumn(String.valueOf(s)))); - } - response.setRows(tableRows); - return response; - } - @Override public DataViewResponse viewData( String name, DataType type, String key, int pageNumber, int itemPerPage) { @@ -351,18 +434,10 @@ public DataViewResponse viewData( if (DataType.isUnknown(type)) { return DataViewResponse.createErrorMessage("Data type is not provided."); } - switch (type) { - case SET: - return responseForSet(clean(name)); - case ZSET: - return responseForZset(clean(name), clean(key), pageNumber, itemPerPage); - case LIST: - return responseForList(clean(name), pageNumber, itemPerPage); - case KEY: - return responseForKeyVal(clean(name)); - default: - throw new UnknownSwitchCase(type.name()); - } + // Delegate the per-type dispatch to the storage layer. Backends without arbitrary keyed + // reads (NATS) throw BackendCapabilityException → 501; the web advice surfaces it. + return messageBrowsingRepository.viewData( + clean(name), type, key == null ? null : clean(key), pageNumber, itemPerPage); } private void setHeadersIfRequired( @@ -390,74 +465,61 @@ private void setHeadersIfRequired( @Override public List> getRunningTasks() { - List queueConfigs = rqueueSystemManagerService.getSortedQueueConfigs(); - List> rows = new ArrayList<>(); - List result = new ArrayList<>(); - if (!CollectionUtils.isEmpty(queueConfigs)) { - result = RedisUtils.executePipeLine( - stringRqueueRedisTemplate.getRedisTemplate(), - ((connection, keySerializer, valueSerializer) -> { - for (QueueConfig queueConfig : queueConfigs) { - connection.zCard(keySerializer.serialize(queueConfig.getProcessingQueueName())); - } - })); - } - rows.add(Arrays.asList("Queue", "Processing [ZSET]", "Number of Messages")); - for (int i = 0; i < queueConfigs.size(); i++) { - QueueConfig queueConfig = queueConfigs.get(i); - rows.add(Arrays.asList( - queueConfig.getName(), queueConfig.getProcessingQueueName(), result.get(i))); - } - return rows; + return bulkSizeTable( + rqueueSystemManagerService.getSortedQueueConfigs(), + QueueConfig::getProcessingQueueName, + DataType.ZSET, + "Processing [ZSET]"); } @Override public List> getWaitingTasks() { - List queueConfigs = rqueueSystemManagerService.getSortedQueueConfigs(); - List> rows = new ArrayList<>(); - List result = new ArrayList<>(); - if (!CollectionUtils.isEmpty(queueConfigs)) { - result = RedisUtils.executePipeLine( - stringRqueueRedisTemplate.getRedisTemplate(), - ((connection, keySerializer, valueSerializer) -> { - for (QueueConfig queueConfig : queueConfigs) { - connection.lLen(keySerializer.serialize(queueConfig.getQueueName())); - } - })); - } - rows.add(Arrays.asList("Queue", "Queue [LIST]", "Number of Messages")); - for (int i = 0; i < queueConfigs.size(); i++) { - QueueConfig queueConfig = queueConfigs.get(i); - rows.add(Arrays.asList(queueConfig.getName(), queueConfig.getQueueName(), result.get(i))); - } - return rows; + return bulkSizeTable( + rqueueSystemManagerService.getSortedQueueConfigs(), + QueueConfig::getQueueName, + DataType.LIST, + "Queue [LIST]"); } @Override public List> getScheduledTasks() { - List queueConfigs = rqueueSystemManagerService.getSortedQueueConfigs(); + return bulkSizeTable( + rqueueSystemManagerService.getSortedQueueConfigs(), + QueueConfig::getScheduledQueueName, + DataType.ZSET, + "Scheduled [ZSET]"); + } + + /** + * Render the home-dashboard "queue / data-name / count" 3-column table for a per-queue data + * structure. The repository's {@link MessageBrowsingRepository#getDataSizes(List, List)} is + * expected to pipeline on Redis; NATS returns zeros. + */ + private List> bulkSizeTable( + List queueConfigs, + java.util.function.Function nameExtractor, + DataType dataType, + String columnLabel) { List> rows = new ArrayList<>(); - List result = new ArrayList<>(); - if (!CollectionUtils.isEmpty(queueConfigs)) { - result = RedisUtils.executePipeLine( - stringRqueueRedisTemplate.getRedisTemplate(), - ((connection, keySerializer, valueSerializer) -> { - for (QueueConfig queueConfig : queueConfigs) { - connection.zCard(keySerializer.serialize(queueConfig.getScheduledQueueName())); - } - })); - } - rows.add(Arrays.asList("Queue", "Scheduled [ZSET]", "Number of Messages")); + rows.add(Arrays.asList("Queue", columnLabel, "Number of Messages")); + if (CollectionUtils.isEmpty(queueConfigs)) { + return rows; + } + List names = new ArrayList<>(queueConfigs.size()); + List types = new ArrayList<>(queueConfigs.size()); + for (QueueConfig queueConfig : queueConfigs) { + names.add(nameExtractor.apply(queueConfig)); + types.add(dataType); + } + List sizes = messageBrowsingRepository.getDataSizes(names, types); for (int i = 0; i < queueConfigs.size(); i++) { - QueueConfig queueConfig = queueConfigs.get(i); - rows.add( - Arrays.asList(queueConfig.getName(), queueConfig.getScheduledQueueName(), result.get(i))); + rows.add(Arrays.asList(queueConfigs.get(i).getName(), names.get(i), sizes.get(i))); } return rows; } private void addRows( - List result, + List result, List> rows, List> queueConfigAndDlq) { for (int i = 0, j = 0; i < queueConfigAndDlq.size(); i++) { @@ -492,17 +554,17 @@ public List> getDeadLetterTasks() { } } List> rows = new ArrayList<>(); - List result = new ArrayList<>(); + List result = new ArrayList<>(); if (!CollectionUtils.isEmpty(queueConfigAndDlq)) { - result = RedisUtils.executePipeLine( - stringRqueueRedisTemplate.getRedisTemplate(), - ((connection, keySerializer, valueSerializer) -> { - for (Entry entry : queueConfigAndDlq) { - if (!entry.getValue().isEmpty()) { - connection.lLen(keySerializer.serialize(entry.getValue())); - } - } - })); + List dlqNames = new ArrayList<>(); + List dlqTypes = new ArrayList<>(); + for (Entry entry : queueConfigAndDlq) { + if (!entry.getValue().isEmpty()) { + dlqNames.add(entry.getValue()); + dlqTypes.add(DataType.LIST); + } + } + result = messageBrowsingRepository.getDataSizes(dlqNames, dlqTypes); } rows.add(Arrays.asList("Queue", "Dead Letter Queues [LIST]", "Number of Messages")); addRows(result, rows, queueConfigAndDlq); diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueSystemManagerServiceImpl.java b/rqueue-web/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueSystemManagerServiceImpl.java similarity index 86% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueSystemManagerServiceImpl.java rename to rqueue-web/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueSystemManagerServiceImpl.java index 3adabd4d2..45e95d753 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueSystemManagerServiceImpl.java +++ b/rqueue-web/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueSystemManagerServiceImpl.java @@ -27,8 +27,8 @@ import com.github.sonus21.rqueue.models.db.QueueConfig; import com.github.sonus21.rqueue.models.event.RqueueBootstrapEvent; import com.github.sonus21.rqueue.models.response.BaseResponse; +import com.github.sonus21.rqueue.service.RqueueMessageMetadataService; import com.github.sonus21.rqueue.utils.RetryableRunnable; -import com.github.sonus21.rqueue.web.service.RqueueMessageMetadataService; import com.github.sonus21.rqueue.web.service.RqueueSystemManagerService; import java.util.ArrayList; import java.util.Arrays; @@ -52,21 +52,31 @@ public class RqueueSystemManagerServiceImpl implements RqueueSystemManagerService { private final RqueueConfig rqueueConfig; - private final RqueueStringDao rqueueStringDao; private final RqueueSystemConfigDao rqueueSystemConfigDao; private final RqueueMessageMetadataService rqueueMessageMetadataService; + + /** + * Redis-only DAO for the per-queue data wipe in {@link #deleteQueue(String)}. {@code required + * = false} so the bean wires on NATS too — there {@code deleteQueue} returns + * {@code code = 1 "not supported"} since JetStream has no equivalent atomic + * "delete-these-keys-and-set-this-config" primitive. Source-of-truth for the queue list now + * comes from {@link EndpointRegistry}, removing the previous Redis-only "set of queue names" + * key. + */ + private final RqueueStringDao rqueueStringDao; + private ScheduledExecutorService executorService; @Autowired public RqueueSystemManagerServiceImpl( RqueueConfig rqueueConfig, - RqueueStringDao rqueueStringDao, RqueueSystemConfigDao rqueueSystemConfigDao, - RqueueMessageMetadataService rqueueMessageMetadataService) { + RqueueMessageMetadataService rqueueMessageMetadataService, + @Autowired(required = false) RqueueStringDao rqueueStringDao) { this.rqueueConfig = rqueueConfig; - this.rqueueStringDao = rqueueStringDao; this.rqueueSystemConfigDao = rqueueSystemConfigDao; this.rqueueMessageMetadataService = rqueueMessageMetadataService; + this.rqueueStringDao = rqueueStringDao; } private List queueKeys(QueueConfig queueConfig) { @@ -92,6 +102,12 @@ public BaseResponse deleteQueue(String queueName) { baseResponse.setMessage("Queue not found"); return baseResponse; } + if (rqueueStringDao == null) { + // NATS path: no equivalent atomic "delete-keys-and-set-config" primitive. + baseResponse.setCode(1); + baseResponse.setMessage("deleteQueue is not supported on rqueue.backend=nats in v1"); + return baseResponse; + } queueConfig.setDeletedOn(System.currentTimeMillis()); queueConfig.setDeleted(true); rqueueStringDao.deleteAndSet( @@ -148,7 +164,9 @@ private void createOrUpdateConfigs(List queueDetails) { for (QueueDetail queueDetail : queueDetails) { queues[i++] = queueDetail.getName(); } - rqueueStringDao.appendToSet(rqueueConfig.getQueuesKey(), queues); + // The previous Redis "set of queue names" cache (rqueueStringDao.appendToSet) is gone — + // EndpointRegistry.getActiveQueues() is the in-memory source of truth and works on every + // backend. The system-config DAO still persists the per-queue metadata below. List ids = Arrays.stream(queues).map(rqueueConfig::getQueueConfigKey).collect(Collectors.toList()); List queueConfigs = rqueueSystemConfigDao.findAllQConfig(ids); @@ -209,7 +227,10 @@ private void handleEventForCleanup(RqueueBootstrapEvent event) { @Override public List getQueues() { - return rqueueStringDao.readFromSet(rqueueConfig.getQueuesKey()); + // EndpointRegistry is the in-memory source of truth for active queue names; identical + // semantics to the previous Redis "set of queue names" key on a single instance, and it + // works on both backends without needing a per-backend Redis-set abstraction. + return new ArrayList<>(EndpointRegistry.getActiveQueues()); } @Override diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueViewControllerServiceImpl.java b/rqueue-web/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueViewControllerServiceImpl.java similarity index 98% rename from rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueViewControllerServiceImpl.java rename to rqueue-web/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueViewControllerServiceImpl.java index ac8b59df3..3e6365e1a 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueViewControllerServiceImpl.java +++ b/rqueue-web/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueViewControllerServiceImpl.java @@ -27,10 +27,10 @@ import com.github.sonus21.rqueue.models.registry.RqueueWorkerPollerView; import com.github.sonus21.rqueue.models.registry.RqueueWorkerView; import com.github.sonus21.rqueue.models.response.RedisDataDetail; +import com.github.sonus21.rqueue.service.RqueueUtilityService; import com.github.sonus21.rqueue.utils.DateTimeUtils; import com.github.sonus21.rqueue.web.service.RqueueQDetailService; import com.github.sonus21.rqueue.web.service.RqueueSystemManagerService; -import com.github.sonus21.rqueue.web.service.RqueueUtilityService; import com.github.sonus21.rqueue.web.service.RqueueViewControllerService; import java.util.ArrayList; import java.util.Arrays; @@ -122,6 +122,8 @@ public void queues(Model model, String xForwardedPrefix, int pageNumber) { queueNameConfigs.sort(Entry.comparingByKey()); model.addAttribute("queues", queueConfigs); model.addAttribute("queueConfigs", queueNameConfigs); + model.addAttribute("storageKicker", rqueueQDetailService.storageKicker()); + model.addAttribute("storageDescription", rqueueQDetailService.storageDescription()); model.addAttribute("currentPage", currentPage); model.addAttribute("totalPages", totalPages); model.addAttribute("hasPreviousPage", currentPage > 1); diff --git a/rqueue-core/src/main/resources/public/rqueue/css/rqueue.css b/rqueue-web/src/main/resources/public/rqueue/css/rqueue.css similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/css/rqueue.css rename to rqueue-web/src/main/resources/public/rqueue/css/rqueue.css diff --git a/rqueue-core/src/main/resources/public/rqueue/img/android-chrome-192x192.png b/rqueue-web/src/main/resources/public/rqueue/img/android-chrome-192x192.png similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/img/android-chrome-192x192.png rename to rqueue-web/src/main/resources/public/rqueue/img/android-chrome-192x192.png diff --git a/rqueue-core/src/main/resources/public/rqueue/img/apple-touch-icon.png b/rqueue-web/src/main/resources/public/rqueue/img/apple-touch-icon.png similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/img/apple-touch-icon.png rename to rqueue-web/src/main/resources/public/rqueue/img/apple-touch-icon.png diff --git a/rqueue-core/src/main/resources/public/rqueue/img/favicon-16x16.png b/rqueue-web/src/main/resources/public/rqueue/img/favicon-16x16.png similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/img/favicon-16x16.png rename to rqueue-web/src/main/resources/public/rqueue/img/favicon-16x16.png diff --git a/rqueue-core/src/main/resources/public/rqueue/img/favicon-32x32.png b/rqueue-web/src/main/resources/public/rqueue/img/favicon-32x32.png similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/img/favicon-32x32.png rename to rqueue-web/src/main/resources/public/rqueue/img/favicon-32x32.png diff --git a/rqueue-core/src/main/resources/public/rqueue/img/favicon.ico b/rqueue-web/src/main/resources/public/rqueue/img/favicon.ico similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/img/favicon.ico rename to rqueue-web/src/main/resources/public/rqueue/img/favicon.ico diff --git a/rqueue-core/src/main/resources/public/rqueue/js/rqueue.js b/rqueue-web/src/main/resources/public/rqueue/js/rqueue.js similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/js/rqueue.js rename to rqueue-web/src/main/resources/public/rqueue/js/rqueue.js diff --git a/rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap-grid.css b/rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap-grid.css similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap-grid.css rename to rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap-grid.css diff --git a/rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap-grid.css.map b/rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap-grid.css.map similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap-grid.css.map rename to rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap-grid.css.map diff --git a/rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap-grid.min.css b/rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap-grid.min.css similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap-grid.min.css rename to rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap-grid.min.css diff --git a/rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap-grid.min.css.map b/rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap-grid.min.css.map similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap-grid.min.css.map rename to rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap-grid.min.css.map diff --git a/rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap-reboot.css b/rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap-reboot.css similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap-reboot.css rename to rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap-reboot.css diff --git a/rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap-reboot.css.map b/rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap-reboot.css.map similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap-reboot.css.map rename to rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap-reboot.css.map diff --git a/rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap-reboot.min.css b/rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap-reboot.min.css similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap-reboot.min.css rename to rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap-reboot.min.css diff --git a/rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap-reboot.min.css.map b/rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap-reboot.min.css.map similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap-reboot.min.css.map rename to rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap-reboot.min.css.map diff --git a/rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap.css b/rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap.css similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap.css rename to rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap.css diff --git a/rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap.css.map b/rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap.css.map similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap.css.map rename to rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap.css.map diff --git a/rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap.min.css b/rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap.min.css similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap.min.css rename to rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap.min.css diff --git a/rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap.min.css.map b/rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap.min.css.map similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap.min.css.map rename to rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/css/bootstrap.min.css.map diff --git a/rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/js/bootstrap.bundle.js b/rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/js/bootstrap.bundle.js similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/js/bootstrap.bundle.js rename to rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/js/bootstrap.bundle.js diff --git a/rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/js/bootstrap.bundle.js.map b/rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/js/bootstrap.bundle.js.map similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/js/bootstrap.bundle.js.map rename to rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/js/bootstrap.bundle.js.map diff --git a/rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/js/bootstrap.bundle.min.js b/rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/js/bootstrap.bundle.min.js similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/js/bootstrap.bundle.min.js rename to rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/js/bootstrap.bundle.min.js diff --git a/rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/js/bootstrap.bundle.min.js.map b/rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/js/bootstrap.bundle.min.js.map similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/js/bootstrap.bundle.min.js.map rename to rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/js/bootstrap.bundle.min.js.map diff --git a/rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/js/bootstrap.js b/rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/js/bootstrap.js similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/js/bootstrap.js rename to rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/js/bootstrap.js diff --git a/rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/js/bootstrap.js.map b/rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/js/bootstrap.js.map similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/js/bootstrap.js.map rename to rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/js/bootstrap.js.map diff --git a/rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/js/bootstrap.min.js b/rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/js/bootstrap.min.js similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/js/bootstrap.min.js rename to rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/js/bootstrap.min.js diff --git a/rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/js/bootstrap.min.js.map b/rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/js/bootstrap.min.js.map similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/vendor/bootstrap/js/bootstrap.min.js.map rename to rqueue-web/src/main/resources/public/rqueue/vendor/bootstrap/js/bootstrap.min.js.map diff --git a/rqueue-core/src/main/resources/public/rqueue/vendor/boxicons/css/animations.css b/rqueue-web/src/main/resources/public/rqueue/vendor/boxicons/css/animations.css similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/vendor/boxicons/css/animations.css rename to rqueue-web/src/main/resources/public/rqueue/vendor/boxicons/css/animations.css diff --git a/rqueue-core/src/main/resources/public/rqueue/vendor/boxicons/css/boxicons.css b/rqueue-web/src/main/resources/public/rqueue/vendor/boxicons/css/boxicons.css similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/vendor/boxicons/css/boxicons.css rename to rqueue-web/src/main/resources/public/rqueue/vendor/boxicons/css/boxicons.css diff --git a/rqueue-core/src/main/resources/public/rqueue/vendor/boxicons/css/boxicons.min.css b/rqueue-web/src/main/resources/public/rqueue/vendor/boxicons/css/boxicons.min.css similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/vendor/boxicons/css/boxicons.min.css rename to rqueue-web/src/main/resources/public/rqueue/vendor/boxicons/css/boxicons.min.css diff --git a/rqueue-core/src/main/resources/public/rqueue/vendor/boxicons/css/transformations.css b/rqueue-web/src/main/resources/public/rqueue/vendor/boxicons/css/transformations.css similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/vendor/boxicons/css/transformations.css rename to rqueue-web/src/main/resources/public/rqueue/vendor/boxicons/css/transformations.css diff --git a/rqueue-core/src/main/resources/public/rqueue/vendor/boxicons/fonts/boxicons.eot b/rqueue-web/src/main/resources/public/rqueue/vendor/boxicons/fonts/boxicons.eot similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/vendor/boxicons/fonts/boxicons.eot rename to rqueue-web/src/main/resources/public/rqueue/vendor/boxicons/fonts/boxicons.eot diff --git a/rqueue-core/src/main/resources/public/rqueue/vendor/boxicons/fonts/boxicons.svg b/rqueue-web/src/main/resources/public/rqueue/vendor/boxicons/fonts/boxicons.svg similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/vendor/boxicons/fonts/boxicons.svg rename to rqueue-web/src/main/resources/public/rqueue/vendor/boxicons/fonts/boxicons.svg diff --git a/rqueue-core/src/main/resources/public/rqueue/vendor/boxicons/fonts/boxicons.ttf b/rqueue-web/src/main/resources/public/rqueue/vendor/boxicons/fonts/boxicons.ttf similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/vendor/boxicons/fonts/boxicons.ttf rename to rqueue-web/src/main/resources/public/rqueue/vendor/boxicons/fonts/boxicons.ttf diff --git a/rqueue-core/src/main/resources/public/rqueue/vendor/boxicons/fonts/boxicons.woff b/rqueue-web/src/main/resources/public/rqueue/vendor/boxicons/fonts/boxicons.woff similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/vendor/boxicons/fonts/boxicons.woff rename to rqueue-web/src/main/resources/public/rqueue/vendor/boxicons/fonts/boxicons.woff diff --git a/rqueue-core/src/main/resources/public/rqueue/vendor/boxicons/fonts/boxicons.woff2 b/rqueue-web/src/main/resources/public/rqueue/vendor/boxicons/fonts/boxicons.woff2 similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/vendor/boxicons/fonts/boxicons.woff2 rename to rqueue-web/src/main/resources/public/rqueue/vendor/boxicons/fonts/boxicons.woff2 diff --git a/rqueue-core/src/main/resources/public/rqueue/vendor/jquery/jquery.min.js b/rqueue-web/src/main/resources/public/rqueue/vendor/jquery/jquery.min.js similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/vendor/jquery/jquery.min.js rename to rqueue-web/src/main/resources/public/rqueue/vendor/jquery/jquery.min.js diff --git a/rqueue-core/src/main/resources/public/rqueue/vendor/jquery/jquery.min.map b/rqueue-web/src/main/resources/public/rqueue/vendor/jquery/jquery.min.map similarity index 100% rename from rqueue-core/src/main/resources/public/rqueue/vendor/jquery/jquery.min.map rename to rqueue-web/src/main/resources/public/rqueue/vendor/jquery/jquery.min.map diff --git a/rqueue-core/src/main/resources/templates/rqueue/base.html b/rqueue-web/src/main/resources/templates/rqueue/base.html similarity index 96% rename from rqueue-core/src/main/resources/templates/rqueue/base.html rename to rqueue-web/src/main/resources/templates/rqueue/base.html index b8841544e..add22e712 100644 --- a/rqueue-core/src/main/resources/templates/rqueue/base.html +++ b/rqueue-web/src/main/resources/templates/rqueue/base.html @@ -59,12 +59,16 @@

Rqueue

Running + {# Hidden when the active broker reports !supportsScheduledIntrospection (e.g. JetStream). + Defaults to visible (hideScheduledPanel == null/false) for the Redis backend. #} + {% if not hideScheduledPanel %}
  • Scheduled
    • + {% endif %}
  • diff --git a/rqueue-core/src/main/resources/templates/rqueue/data_explorer_modal.html b/rqueue-web/src/main/resources/templates/rqueue/data_explorer_modal.html similarity index 100% rename from rqueue-core/src/main/resources/templates/rqueue/data_explorer_modal.html rename to rqueue-web/src/main/resources/templates/rqueue/data_explorer_modal.html diff --git a/rqueue-core/src/main/resources/templates/rqueue/index.html b/rqueue-web/src/main/resources/templates/rqueue/index.html similarity index 100% rename from rqueue-core/src/main/resources/templates/rqueue/index.html rename to rqueue-web/src/main/resources/templates/rqueue/index.html diff --git a/rqueue-core/src/main/resources/templates/rqueue/latency_chart.html b/rqueue-web/src/main/resources/templates/rqueue/latency_chart.html similarity index 100% rename from rqueue-core/src/main/resources/templates/rqueue/latency_chart.html rename to rqueue-web/src/main/resources/templates/rqueue/latency_chart.html diff --git a/rqueue-core/src/main/resources/templates/rqueue/queue_detail.html b/rqueue-web/src/main/resources/templates/rqueue/queue_detail.html similarity index 98% rename from rqueue-core/src/main/resources/templates/rqueue/queue_detail.html rename to rqueue-web/src/main/resources/templates/rqueue/queue_detail.html index 6610c9259..7e97f9348 100644 --- a/rqueue-core/src/main/resources/templates/rqueue/queue_detail.html +++ b/rqueue-web/src/main/resources/templates/rqueue/queue_detail.html @@ -136,6 +136,7 @@

    Queue Pollers

    Worker Id + Consumer Name Status Host Pid @@ -152,6 +153,7 @@

    Queue Pollers

    {% for worker in queueWorkers %} {{worker.workerId}} + {% if worker.consumerName %}{{worker.consumerName}}{% endif %} {{worker.status}} {{worker.host}} {{worker.pid}} diff --git a/rqueue-core/src/main/resources/templates/rqueue/queues.html b/rqueue-web/src/main/resources/templates/rqueue/queues.html similarity index 98% rename from rqueue-core/src/main/resources/templates/rqueue/queues.html rename to rqueue-web/src/main/resources/templates/rqueue/queues.html index e2b17b6ff..de4d7fcd9 100644 --- a/rqueue-core/src/main/resources/templates/rqueue/queues.html +++ b/rqueue-web/src/main/resources/templates/rqueue/queues.html @@ -170,10 +170,10 @@

    {{meta.name}}
    - Redis Layout + {{storageKicker}} Layout

    Queue Storage Footprint

    -

    Underlying Redis structures for the queues visible on this page.

    +

    {{storageDescription}}

    diff --git a/rqueue-core/src/main/resources/templates/rqueue/running.html b/rqueue-web/src/main/resources/templates/rqueue/running.html similarity index 100% rename from rqueue-core/src/main/resources/templates/rqueue/running.html rename to rqueue-web/src/main/resources/templates/rqueue/running.html diff --git a/rqueue-core/src/main/resources/templates/rqueue/stats_chart.html b/rqueue-web/src/main/resources/templates/rqueue/stats_chart.html similarity index 100% rename from rqueue-core/src/main/resources/templates/rqueue/stats_chart.html rename to rqueue-web/src/main/resources/templates/rqueue/stats_chart.html diff --git a/rqueue-core/src/main/resources/templates/rqueue/utility.html b/rqueue-web/src/main/resources/templates/rqueue/utility.html similarity index 100% rename from rqueue-core/src/main/resources/templates/rqueue/utility.html rename to rqueue-web/src/main/resources/templates/rqueue/utility.html diff --git a/rqueue-core/src/main/resources/templates/rqueue/workers.html b/rqueue-web/src/main/resources/templates/rqueue/workers.html similarity index 97% rename from rqueue-core/src/main/resources/templates/rqueue/workers.html rename to rqueue-web/src/main/resources/templates/rqueue/workers.html index 3c6b7464a..222921124 100644 --- a/rqueue-core/src/main/resources/templates/rqueue/workers.html +++ b/rqueue-web/src/main/resources/templates/rqueue/workers.html @@ -105,6 +105,9 @@

    {{worker.workerId}}

    {{poller.queue}}

    + {% if poller.consumerName %} + {{poller.consumerName}} + {% endif %}
    > details = service.getQueueDataStructureDetail(queueConfig); + + RedisDataDetail pending = details.stream() + .filter(e -> e.getKey() == NavTab.PENDING) + .findFirst() + .orElseThrow() + .getValue(); + assertEquals(42L, pending.getSize()); + verify(messageBroker, atLeastOnce()).size(any(QueueDetail.class)); + verify(messageBrowsingRepository, never()) + .getDataSize( + queueConfig.getQueueName(), com.github.sonus21.rqueue.models.enums.DataType.LIST); + } + + @Test + void sizeFallsBackToRedisWhenNoBroker() { + when(messageBrowsingRepository.getDataSize( + queueConfig.getQueueName(), com.github.sonus21.rqueue.models.enums.DataType.LIST)) + .thenReturn(7L); + when(messageBrowsingRepository.getDataSize( + queueConfig.getProcessingQueueName(), + com.github.sonus21.rqueue.models.enums.DataType.ZSET)) + .thenReturn(0L); + when(messageBrowsingRepository.getDataSize( + queueConfig.getScheduledQueueName(), + com.github.sonus21.rqueue.models.enums.DataType.ZSET)) + .thenReturn(0L); + + List> details = service.getQueueDataStructureDetail(queueConfig); + + RedisDataDetail pending = details.stream() + .filter(e -> e.getKey() == NavTab.PENDING) + .findFirst() + .orElseThrow() + .getValue(); + assertEquals(7L, pending.getSize()); + } + + @Test + void scheduledAndRunningTabsHiddenForNatsBroker() { + // NATS: supportsScheduledIntrospection=false, usesPrimaryHandlerDispatch=false + Capabilities natsCaps = new Capabilities(true, false, false, false); + service.setMessageBroker(messageBroker); + when(messageBroker.capabilities()).thenReturn(natsCaps); + when(messageBroker.size(any(QueueDetail.class))).thenReturn(0L); + + List> details = service.getQueueDataStructureDetail(queueConfig); + assertFalse( + details.stream().anyMatch(e -> e.getKey() == NavTab.SCHEDULED), + "scheduled nav tab should be hidden for NATS"); + assertFalse( + details.stream().anyMatch(e -> e.getKey() == NavTab.RUNNING), + "running nav tab should be hidden for NATS (no processing ZSET)"); + + List tabs = service.getNavTabs(queueConfig); + assertFalse(tabs.contains(NavTab.SCHEDULED)); + assertFalse(tabs.contains(NavTab.RUNNING)); + + when(rqueueSystemManagerService.getQueueConfig(queueConfig.getName())).thenReturn(queueConfig); + DataViewResponse explore = service.getExplorePageData( + queueConfig.getName(), queueConfig.getScheduledQueueName(), DataType.ZSET, 0, 10); + assertTrue(explore.isHideScheduledPanel()); + assertTrue(explore.isHideCronJobs()); + assertTrue(explore.getRows() == null || explore.getRows().isEmpty()); + } + + @Test + void peekRoutesThroughBrokerForReadyList() { + service.setMessageBroker(messageBroker); + when(messageBroker.capabilities()).thenReturn(Capabilities.REDIS_DEFAULTS); + when(messageBroker.peek(any(QueueDetail.class), anyLong(), anyLong())) + .thenReturn(Collections.emptyList()); + when(rqueueSystemManagerService.getQueueConfig(queueConfig.getName())).thenReturn(queueConfig); + + DataViewResponse response = service.getExplorePageData( + queueConfig.getName(), queueConfig.getQueueName(), DataType.LIST, 0, 10); + + verify(messageBroker, atLeastOnce()).peek(any(QueueDetail.class), anyLong(), anyLong()); + assertTrue(response.getRows() == null || response.getRows().isEmpty()); + } + + @Test + void hideFlagsDefaultFalseWithRedisBroker() { + service.setMessageBroker(messageBroker); + when(messageBroker.capabilities()).thenReturn(Capabilities.REDIS_DEFAULTS); + when(messageBroker.peek(any(QueueDetail.class), anyLong(), anyLong())) + .thenReturn(Collections.emptyList()); + when(rqueueSystemManagerService.getQueueConfig(queueConfig.getName())).thenReturn(queueConfig); + + DataViewResponse response = service.getExplorePageData( + queueConfig.getName(), queueConfig.getQueueName(), DataType.LIST, 0, 10); + + assertFalse(response.isHideScheduledPanel()); + assertFalse(response.isHideCronJobs()); + } +} diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/web/service/RqueueQDetailServiceTest.java b/rqueue-web/src/test/java/com/github/sonus21/rqueue/web/service/RqueueQDetailServiceTest.java similarity index 77% rename from rqueue-core/src/test/java/com/github/sonus21/rqueue/web/service/RqueueQDetailServiceTest.java rename to rqueue-web/src/test/java/com/github/sonus21/rqueue/web/service/RqueueQDetailServiceTest.java index 9521b86c1..19cf31546 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/web/service/RqueueQDetailServiceTest.java +++ b/rqueue-web/src/test/java/com/github/sonus21/rqueue/web/service/RqueueQDetailServiceTest.java @@ -17,16 +17,14 @@ package com.github.sonus21.rqueue.web.service; import static com.github.sonus21.rqueue.utils.TestUtils.createQueueConfig; -import static com.google.common.collect.Lists.newArrayList; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyCollection; +import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.Mockito.doReturn; import com.github.sonus21.TestBase; import com.github.sonus21.rqueue.CoreUnitTest; -import com.github.sonus21.rqueue.common.RqueueRedisTemplate; import com.github.sonus21.rqueue.config.RqueueConfig; import com.github.sonus21.rqueue.converter.GenericMessageConverter; import com.github.sonus21.rqueue.core.RqueueMessage; @@ -47,6 +45,8 @@ import com.github.sonus21.rqueue.models.response.RowColumnMetaType; import com.github.sonus21.rqueue.models.response.TableColumn; import com.github.sonus21.rqueue.models.response.TableRow; +import com.github.sonus21.rqueue.repository.MessageBrowsingRepository; +import com.github.sonus21.rqueue.service.RqueueMessageMetadataService; import com.github.sonus21.rqueue.utils.RqueueMessageTestUtils; import com.github.sonus21.rqueue.web.service.impl.RqueueQDetailServiceImpl; import com.github.sonus21.rqueue.worker.RqueueWorkerRegistry; @@ -55,20 +55,16 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.Set; import java.util.stream.Collectors; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mock; import org.mockito.MockitoAnnotations; import org.springframework.data.redis.core.DefaultTypedTuple; -import org.springframework.data.redis.core.RedisCallback; import org.springframework.data.redis.core.RedisTemplate; -import org.springframework.data.redis.core.ZSetOperations.TypedTuple; import org.springframework.messaging.converter.MessageConverter; @CoreUnitTest @@ -80,7 +76,7 @@ class RqueueQDetailServiceTest extends TestBase { private RedisTemplate redisTemplate; @Mock - private RqueueRedisTemplate stringRqueueRedisTemplate; + private MessageBrowsingRepository messageBrowsingRepository; @Mock private RqueueMessageTemplate rqueueMessageTemplate; @@ -105,7 +101,7 @@ class RqueueQDetailServiceTest extends TestBase { public void init() { MockitoAnnotations.openMocks(this); rqueueQDetailService = new RqueueQDetailServiceImpl( - stringRqueueRedisTemplate, + messageBrowsingRepository, rqueueMessageTemplate, rqueueSystemManagerService, rqueueMessageMetadataService, @@ -120,10 +116,18 @@ public void init() { @Test void getQueueDataStructureDetail() { assertEquals(Collections.emptyList(), rqueueQDetailService.getQueueDataStructureDetail(null)); - doReturn(10L).when(stringRqueueRedisTemplate).getListSize("__rq::queue::test"); - doReturn(11L).when(stringRqueueRedisTemplate).getListSize("test-dlq"); - doReturn(12L).when(stringRqueueRedisTemplate).getZsetSize("__rq::d-queue::test"); - doReturn(5L).when(stringRqueueRedisTemplate).getZsetSize("__rq::p-queue::test"); + doReturn(10L) + .when(messageBrowsingRepository) + .getDataSize("__rq::queue::test", com.github.sonus21.rqueue.models.enums.DataType.LIST); + doReturn(11L) + .when(messageBrowsingRepository) + .getDataSize("test-dlq", com.github.sonus21.rqueue.models.enums.DataType.LIST); + doReturn(12L) + .when(messageBrowsingRepository) + .getDataSize("__rq::d-queue::test", com.github.sonus21.rqueue.models.enums.DataType.ZSET); + doReturn(5L) + .when(messageBrowsingRepository) + .getDataSize("__rq::p-queue::test", com.github.sonus21.rqueue.models.enums.DataType.ZSET); List> queueRedisDataDetails = new ArrayList<>(); queueRedisDataDetails.add(new HashMap.SimpleEntry<>( NavTab.PENDING, new RedisDataDetail("__rq::queue::test", DataType.LIST, 10))); @@ -143,10 +147,18 @@ void getQueueDataStructureDetail() { @Test void getQueueDataStructureDetails() { - doReturn(10L).when(stringRqueueRedisTemplate).getListSize("__rq::queue::test"); - doReturn(11L).when(stringRqueueRedisTemplate).getListSize("test-dlq"); - doReturn(12L).when(stringRqueueRedisTemplate).getZsetSize("__rq::d-queue::test"); - doReturn(5L).when(stringRqueueRedisTemplate).getZsetSize("__rq::p-queue::test"); + doReturn(10L) + .when(messageBrowsingRepository) + .getDataSize("__rq::queue::test", com.github.sonus21.rqueue.models.enums.DataType.LIST); + doReturn(11L) + .when(messageBrowsingRepository) + .getDataSize("test-dlq", com.github.sonus21.rqueue.models.enums.DataType.LIST); + doReturn(12L) + .when(messageBrowsingRepository) + .getDataSize("__rq::d-queue::test", com.github.sonus21.rqueue.models.enums.DataType.ZSET); + doReturn(5L) + .when(messageBrowsingRepository) + .getDataSize("__rq::p-queue::test", com.github.sonus21.rqueue.models.enums.DataType.ZSET); List> queueRedisDataDetails = new ArrayList<>(); queueRedisDataDetails.add(new HashMap.SimpleEntry<>( NavTab.PENDING, new RedisDataDetail("__rq::queue::test", DataType.LIST, 10))); @@ -161,9 +173,15 @@ void getQueueDataStructureDetails() { DataType.LIST, 11))); - doReturn(5L).when(stringRqueueRedisTemplate).getListSize("__rq::queue::test2"); - doReturn(2L).when(stringRqueueRedisTemplate).getZsetSize("__rq::p-queue::test2"); - doReturn(8L).when(stringRqueueRedisTemplate).getZsetSize("__rq::d-queue::test2"); + doReturn(5L) + .when(messageBrowsingRepository) + .getDataSize("__rq::queue::test2", com.github.sonus21.rqueue.models.enums.DataType.LIST); + doReturn(2L) + .when(messageBrowsingRepository) + .getDataSize("__rq::p-queue::test2", com.github.sonus21.rqueue.models.enums.DataType.ZSET); + doReturn(8L) + .when(messageBrowsingRepository) + .getDataSize("__rq::d-queue::test2", com.github.sonus21.rqueue.models.enums.DataType.ZSET); List> queueRedisDataDetails2 = new ArrayList<>(); queueRedisDataDetails2.add(new HashMap.SimpleEntry<>( @@ -398,107 +416,19 @@ void getExplorePageDataTypeProcessingQueue() { assertEquals(expectedResponse, response); } + // Per-type rendering for viewData (KEY / LIST / ZSET / SET) is now exercised inside + // RedisMessageBrowsingRepository (the storage layer); the service is a thin pass-through. + // Below we verify the service correctly forwards arguments to the repository and returns + // its response unchanged. Detailed per-type rendering coverage belongs in a future + // RedisMessageBrowsingRepositoryTest. @Test - void viewDataKey() { - doReturn("test").when(stringRqueueRedisTemplate).get("key"); - DataViewResponse response = rqueueQDetailService.viewData("key", DataType.KEY, null, 0, 10); - DataViewResponse expectedResponse = new DataViewResponse(); - expectedResponse.setHeaders(Collections.singletonList("Value")); - expectedResponse.setRows(Collections.singletonList(new TableRow(new TableColumn("test")))); - assertEquals(expectedResponse, response); - - doReturn(null).when(stringRqueueRedisTemplate).get("key2"); - response = rqueueQDetailService.viewData("key2", DataType.KEY, null, 0, 10); - expectedResponse.setRows(Collections.singletonList(new TableRow(new TableColumn("null")))); - assertEquals(expectedResponse, response); - } - - @Test - void viewDataList() { - List objects = new ArrayList<>(); - objects.add("Test"); - objects.add(RqueueMessageUtils.buildMessage( - RqueueMessageTestUtils.MESSAGE_ID_GENERATOR, - messageConverter, - "jobs", - null, - "buildMessage", - null, - null, - null)); - objects.add(null); - doReturn(objects).when(stringRqueueRedisTemplate).lrange("jobs", 0, 9); + void viewDataDelegatesToRepository() { + DataViewResponse stub = new DataViewResponse(); + stub.setHeaders(Collections.singletonList("Item")); + stub.setRows(Collections.singletonList(new TableRow(new TableColumn("hello")))); + doReturn(stub).when(messageBrowsingRepository).viewData("jobs", DataType.LIST, null, 0, 10); DataViewResponse response = rqueueQDetailService.viewData("jobs", DataType.LIST, null, 0, 10); - DataViewResponse expectedResponse = new DataViewResponse(); - expectedResponse.setHeaders(Collections.singletonList("Item")); - List tableRows = new ArrayList<>(); - for (Object o : objects) { - tableRows.add(new TableRow(new TableColumn(String.valueOf(o)))); - } - expectedResponse.setRows(tableRows); - assertEquals(expectedResponse, response); - } - - @Test - void viewDataZset() { - Set> objects = new HashSet<>(); - objects.add(new DefaultTypedTuple<>("Test", 100.0)); - objects.add(new DefaultTypedTuple<>( - RqueueMessageUtils.buildMessage( - RqueueMessageTestUtils.MESSAGE_ID_GENERATOR, - messageConverter, - "jobs", - null, - "buildMessage", - null, - null, - null), - 200.0)); - - List tableRows = new ArrayList<>(); - for (TypedTuple typedTuple : objects) { - List items = new ArrayList<>(); - items.add(new TableColumn(String.valueOf(typedTuple.getValue()))); - items.add(new TableColumn(typedTuple.getScore())); - tableRows.add(new TableRow(items)); - } - DataViewResponse expectedResponse = new DataViewResponse(); - List headers = new ArrayList<>(); - headers.add("Value"); - headers.add("Score"); - expectedResponse.setHeaders(headers); - - expectedResponse.setRows(tableRows); - - doReturn(objects).when(stringRqueueRedisTemplate).zrangeWithScore("jobs", 0, 9); - DataViewResponse response = rqueueQDetailService.viewData("jobs", DataType.ZSET, null, 0, 10); - - assertEquals(expectedResponse, response); - } - - @Test - void viewDataSet() { - Set objects = new HashSet<>(); - objects.add("Test"); - objects.add(RqueueMessageUtils.buildMessage( - RqueueMessageTestUtils.MESSAGE_ID_GENERATOR, - messageConverter, - "jobs", - null, - "Test object", - null, - null, - null)); - List tableRows = new ArrayList<>(); - for (Object object : objects) { - tableRows.add(new TableRow(new TableColumn(String.valueOf(object)))); - } - DataViewResponse expectedResponse = new DataViewResponse(); - expectedResponse.setHeaders(Collections.singletonList("Item")); - expectedResponse.setRows(tableRows); - doReturn(objects).when(stringRqueueRedisTemplate).getMembers("jobs"); - DataViewResponse response = rqueueQDetailService.viewData("jobs", DataType.SET, null, 0, 10); - assertEquals(expectedResponse, response); + assertEquals(stub, response); } @Test @@ -514,7 +444,6 @@ void viewData() { @Test void getScheduledTasks() { - doReturn(redisTemplate).when(stringRqueueRedisTemplate).getRedisTemplate(); QueueConfig queueConfig = createQueueConfig("test", 10, 10000L, null); queueConfig.addDeadLetterQueue(new DeadLetterQueue("test-dlq", false)); QueueConfig queueConfig2 = createQueueConfig("test2", 10, 10000L, null); @@ -524,9 +453,9 @@ void getScheduledTasks() { .when(rqueueSystemManagerService) .getSortedQueueConfigs(); - doReturn(newArrayList(100L, 200L)) - .when(redisTemplate) - .executePipelined(any(RedisCallback.class)); + doReturn(Arrays.asList(100L, 200L)) + .when(messageBrowsingRepository) + .getDataSizes(anyList(), anyList()); List> response = rqueueQDetailService.getScheduledTasks(); assertEquals(3, response.size()); List> expectedResponse = new ArrayList<>(); @@ -541,11 +470,10 @@ void getScheduledTasks() { @Test void getWaitingTasks() { - doReturn(redisTemplate).when(stringRqueueRedisTemplate).getRedisTemplate(); doReturn(queueConfigList).when(rqueueSystemManagerService).getSortedQueueConfigs(); doReturn(Arrays.asList(100L, 110L)) - .when(redisTemplate) - .executePipelined(any(RedisCallback.class)); + .when(messageBrowsingRepository) + .getDataSizes(anyList(), anyList()); List> response = rqueueQDetailService.getWaitingTasks(); assertEquals(3, response.size()); List headers = Arrays.asList("Queue", "Queue [LIST]", "Number of Messages"); @@ -556,11 +484,10 @@ void getWaitingTasks() { @Test void getRunningTasks() { - doReturn(redisTemplate).when(stringRqueueRedisTemplate).getRedisTemplate(); doReturn(queueConfigList).when(rqueueSystemManagerService).getSortedQueueConfigs(); doReturn(Arrays.asList(100L, 110L)) - .when(redisTemplate) - .executePipelined(any(RedisCallback.class)); + .when(messageBrowsingRepository) + .getDataSizes(anyList(), anyList()); List> response = rqueueQDetailService.getRunningTasks(); assertEquals(3, response.size()); List headers = Arrays.asList("Queue", "Processing [ZSET]", "Number of Messages"); @@ -573,11 +500,10 @@ void getRunningTasks() { @Test void getDeadLetterTasks() { - doReturn(redisTemplate).when(stringRqueueRedisTemplate).getRedisTemplate(); doReturn(queueConfigList).when(rqueueSystemManagerService).getSortedQueueConfigs(); doReturn(Arrays.asList(100L, 110L)) - .when(redisTemplate) - .executePipelined(any(RedisCallback.class)); + .when(messageBrowsingRepository) + .getDataSizes(anyList(), anyList()); List> response = rqueueQDetailService.getDeadLetterTasks(); assertEquals(3, response.size()); List headers = diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/web/service/impl/RqueueSystemManagerServiceImplTest.java b/rqueue-web/src/test/java/com/github/sonus21/rqueue/web/service/RqueueSystemManagerServiceImplTest.java similarity index 89% rename from rqueue-core/src/test/java/com/github/sonus21/rqueue/web/service/impl/RqueueSystemManagerServiceImplTest.java rename to rqueue-web/src/test/java/com/github/sonus21/rqueue/web/service/RqueueSystemManagerServiceImplTest.java index af45e25f2..620b7dce5 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/web/service/impl/RqueueSystemManagerServiceImplTest.java +++ b/rqueue-web/src/test/java/com/github/sonus21/rqueue/web/service/RqueueSystemManagerServiceImplTest.java @@ -14,18 +14,15 @@ * */ -package com.github.sonus21.rqueue.web.service.impl; +package com.github.sonus21.rqueue.web.service; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyCollection; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.verifyNoInteractions; @@ -39,8 +36,9 @@ import com.github.sonus21.rqueue.listener.QueueDetail; import com.github.sonus21.rqueue.models.db.QueueConfig; import com.github.sonus21.rqueue.models.event.RqueueBootstrapEvent; +import com.github.sonus21.rqueue.service.RqueueMessageMetadataService; import com.github.sonus21.rqueue.utils.TestUtils; -import com.github.sonus21.rqueue.web.service.RqueueMessageMetadataService; +import com.github.sonus21.rqueue.web.service.impl.RqueueSystemManagerServiceImpl; import java.util.Arrays; import java.util.List; import org.junit.jupiter.api.BeforeEach; @@ -81,7 +79,7 @@ public void init() { MockitoAnnotations.openMocks(this); EndpointRegistry.delete(); rqueueSystemManagerService = new RqueueSystemManagerServiceImpl( - rqueueConfig, rqueueStringDao, rqueueSystemConfigDao, rqueueMessageMetadataService); + rqueueConfig, rqueueSystemConfigDao, rqueueMessageMetadataService, rqueueStringDao); slowQueueConfig.setId(TestUtils.getQueueConfigKey(slowQueue)); fastQueueConfig.setId(TestUtils.getQueueConfigKey(fastQueue)); EndpointRegistry.register(slowQueueDetail); @@ -119,7 +117,6 @@ public void verifyConfigData(QueueConfig expectedConfig, QueueConfig queueConfig @Test void onApplicationEventStartCreateAllQueueConfigs() { - doReturn("__rq::queues").when(rqueueConfig).getQueuesKey(); doAnswer(invocation -> { String name = invocation.getArgument(0); return "__rq::q-config::" + name; @@ -127,18 +124,9 @@ void onApplicationEventStartCreateAllQueueConfigs() { .when(rqueueConfig) .getQueueConfigKey(anyString()); RqueueBootstrapEvent event = new RqueueBootstrapEvent("Container", true); - doAnswer(invocation -> { - if (slowQueue.equals(invocation.getArgument(1))) { - assertEquals(fastQueue, invocation.getArgument(2)); - } else if (fastQueue.equals(invocation.getArgument(1))) { - assertEquals(slowQueue, invocation.getArgument(2)); - } else { - fail(); - } - return 2L; - }) - .when(rqueueStringDao) - .appendToSet(eq(TestUtils.getQueuesKey()), any()); + // Note: rqueueStringDao.appendToSet(...) was removed from the impl as part of the + // backend-neutral refactor (queue names now come from EndpointRegistry instead of a + // Redis-only set). Only saveAllQConfig is exercised on the create path. doAnswer(invocation -> { List queueConfigs = invocation.getArgument(0); assertEquals(2, queueConfigs.size()); diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/web/service/RqueueSystemManagerServiceTest.java b/rqueue-web/src/test/java/com/github/sonus21/rqueue/web/service/RqueueSystemManagerServiceTest.java similarity index 84% rename from rqueue-core/src/test/java/com/github/sonus21/rqueue/web/service/RqueueSystemManagerServiceTest.java rename to rqueue-web/src/test/java/com/github/sonus21/rqueue/web/service/RqueueSystemManagerServiceTest.java index 5f67363e2..b74b79418 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/web/service/RqueueSystemManagerServiceTest.java +++ b/rqueue-web/src/test/java/com/github/sonus21/rqueue/web/service/RqueueSystemManagerServiceTest.java @@ -27,14 +27,15 @@ import com.github.sonus21.TestBase; import com.github.sonus21.rqueue.CoreUnitTest; import com.github.sonus21.rqueue.config.RqueueConfig; +import com.github.sonus21.rqueue.core.EndpointRegistry; import com.github.sonus21.rqueue.dao.RqueueStringDao; import com.github.sonus21.rqueue.dao.RqueueSystemConfigDao; import com.github.sonus21.rqueue.listener.QueueDetail; import com.github.sonus21.rqueue.models.db.QueueConfig; import com.github.sonus21.rqueue.models.response.BaseResponse; +import com.github.sonus21.rqueue.service.RqueueMessageMetadataService; import com.github.sonus21.rqueue.utils.TestUtils; import com.github.sonus21.rqueue.web.service.impl.RqueueSystemManagerServiceImpl; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; @@ -74,8 +75,11 @@ class RqueueSystemManagerServiceTest extends TestBase { @BeforeEach public void init() { MockitoAnnotations.openMocks(this); + // EndpointRegistry is a global static; clear it so other test classes that register + // queues don't leak state into these tests (the new getQueues() reads from it). + EndpointRegistry.delete(); rqueueSystemManagerService = new RqueueSystemManagerServiceImpl( - rqueueConfig, rqueueStringDao, rqueueSystemConfigDao, rqueueMessageMetadataService); + rqueueConfig, rqueueSystemConfigDao, rqueueMessageMetadataService, rqueueStringDao); queues = new HashSet<>(); queues.add(slowQueue); queues.add(fastQueue); @@ -98,12 +102,9 @@ void deleteQueue() { @Test void getQueues() { - doReturn("__rq::queues").when(rqueueConfig).getQueuesKey(); - doReturn(Collections.emptyList()).when(rqueueStringDao).readFromSet(TestUtils.getQueuesKey()); + // The new impl reads active queue names from EndpointRegistry rather than a Redis SET. assertEquals(Collections.emptyList(), rqueueSystemManagerService.getQueues()); - doReturn(Collections.singletonList("job")) - .when(rqueueStringDao) - .readFromSet(TestUtils.getQueuesKey()); + EndpointRegistry.register(TestUtils.createQueueDetail("job")); assertEquals(Collections.singletonList("job"), rqueueSystemManagerService.getQueues()); } @@ -115,12 +116,13 @@ void getQueueConfigs() { }) .when(rqueueConfig) .getQueueConfigKey(anyString()); - doReturn("__rq::queues").when(rqueueConfig).getQueuesKey(); - doReturn(new ArrayList<>(queues)).when(rqueueStringDao).readFromSet(TestUtils.getQueuesKey()); + EndpointRegistry.register(slowQueueDetail); + EndpointRegistry.register(fastQueueDetail); doReturn(Arrays.asList(slowQueueConfig, fastQueueConfig)) .when(rqueueSystemConfigDao) - .findAllQConfig( - queues.stream().map(TestUtils::getQueueConfigKey).collect(Collectors.toList())); + .findAllQConfig(EndpointRegistry.getActiveQueues().stream() + .map(TestUtils::getQueueConfigKey) + .collect(Collectors.toList())); assertEquals( Arrays.asList(slowQueueConfig, fastQueueConfig), rqueueSystemManagerService.getQueueConfigs()); @@ -134,13 +136,12 @@ void getSortedQueueConfigs() { }) .when(rqueueConfig) .getQueueConfigKey(anyString()); - doReturn("__rq::queues").when(rqueueConfig).getQueuesKey(); - doReturn(new ArrayList<>(queues)).when(rqueueStringDao).readFromSet(TestUtils.getQueuesKey()); + EndpointRegistry.register(slowQueueDetail); + EndpointRegistry.register(fastQueueDetail); doReturn(Arrays.asList(slowQueueConfig, fastQueueConfig)) .when(rqueueSystemConfigDao) - .findAllQConfig(queues.stream() + .findAllQConfig(EndpointRegistry.getActiveQueues().stream() .map(TestUtils::getQueueConfigKey) - .sorted() .collect(Collectors.toList())); assertEquals( Arrays.asList(fastQueueConfig, slowQueueConfig), diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/web/service/RqueueTaskMetricsAggregatorServiceTest.java b/rqueue-web/src/test/java/com/github/sonus21/rqueue/web/service/RqueueTaskMetricsAggregatorServiceTest.java similarity index 96% rename from rqueue-core/src/test/java/com/github/sonus21/rqueue/web/service/RqueueTaskMetricsAggregatorServiceTest.java rename to rqueue-web/src/test/java/com/github/sonus21/rqueue/web/service/RqueueTaskMetricsAggregatorServiceTest.java index feaefb728..272714744 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/web/service/RqueueTaskMetricsAggregatorServiceTest.java +++ b/rqueue-web/src/test/java/com/github/sonus21/rqueue/web/service/RqueueTaskMetricsAggregatorServiceTest.java @@ -31,7 +31,7 @@ import com.github.sonus21.rqueue.config.RqueueWebConfig; import com.github.sonus21.rqueue.core.Job; import com.github.sonus21.rqueue.core.RqueueMessage; -import com.github.sonus21.rqueue.core.RqueueMessageTemplate; +import com.github.sonus21.rqueue.core.spi.MessageBroker; import com.github.sonus21.rqueue.dao.RqueueJobDao; import com.github.sonus21.rqueue.dao.RqueueQStatsDao; import com.github.sonus21.rqueue.exception.TimedOutException; @@ -40,9 +40,10 @@ import com.github.sonus21.rqueue.models.aggregator.TasksStat; import com.github.sonus21.rqueue.models.db.MessageMetadata; import com.github.sonus21.rqueue.models.db.QueueStatistics; -import com.github.sonus21.rqueue.models.db.QueueStatisticsTest; +import com.github.sonus21.rqueue.models.db.QueueStatisticsFixtures; import com.github.sonus21.rqueue.models.enums.MessageStatus; import com.github.sonus21.rqueue.models.event.RqueueExecutionEvent; +import com.github.sonus21.rqueue.service.RqueueMessageMetadataService; import com.github.sonus21.rqueue.utils.Constants; import com.github.sonus21.rqueue.utils.DateTimeUtils; import com.github.sonus21.rqueue.utils.TestUtils; @@ -85,7 +86,7 @@ class RqueueTaskMetricsAggregatorServiceTest extends TestBase { private RqueueJobDao rqueueJobDao; @Mock - private RqueueMessageTemplate rqueueMessageTemplate; + private MessageBroker messageBroker; private RqueueJobMetricsAggregatorService rqueueJobMetricsAggregatorService; @@ -124,7 +125,7 @@ private RqueueExecutionEvent generateTaskEventWithStatus(MessageStatus status) { rqueueConfig, rqueueMessageMetadataService, rqueueJobDao, - rqueueMessageTemplate, + messageBroker, rqueueLockManager, queueDetail, messageMetadata, @@ -234,8 +235,8 @@ void onApplicationEvent() throws TimedOutException { QueueStatistics statistics = queueStatistics.get(0); String date = DateTimeUtils.today().toString(); assertEquals(statistics.getId(), id); - QueueStatisticsTest.validate(statistics, 1); - QueueStatisticsTest.checkNonNull(statistics, date); + QueueStatisticsFixtures.validate(statistics, 1); + QueueStatisticsFixtures.checkNonNull(statistics, date); assertEquals(tasksStat.jobRunTime(), statistics.jobRunTime(date)); assertEquals(tasksStat.discarded, statistics.tasksDiscarded(date)); assertEquals(tasksStat.success, statistics.tasksSuccessful(date)); diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/web/view/DateTimeFunctionTest.java b/rqueue-web/src/test/java/com/github/sonus21/rqueue/web/view/DateTimeFunctionTest.java similarity index 100% rename from rqueue-core/src/test/java/com/github/sonus21/rqueue/web/view/DateTimeFunctionTest.java rename to rqueue-web/src/test/java/com/github/sonus21/rqueue/web/view/DateTimeFunctionTest.java diff --git a/settings.gradle b/settings.gradle index 29f234e52..e2c2557e5 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,10 +1,14 @@ rootProject.name = "Rqueue" include "rqueue-test-util" include "rqueue-core" +include "rqueue-redis" +include "rqueue-nats" +include "rqueue-web" include "rqueue-spring-common-test" include "rqueue-spring" include "rqueue-spring-boot-starter" include "rqueue-spring-example" include "rqueue-spring-boot-example" +include "rqueue-spring-boot-nats-example" include "rqueue-spring-boot-reactive-example"