Add graceful restart to client

build.gradle

@@ -34,6 +34,7 @@ nohttp {
 	source.exclude "**/build/**"
 	source.exclude "**/out/**"
 	source.exclude "**/target/**"
+	source.exclude "**/*.dylib"
 }
 
 check {

spring-pulsar-reactive/src/main/java/org/springframework/pulsar/reactive/core/DefaultReactivePulsarSenderFactory.java

@@ -19,6 +19,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
+import java.util.concurrent.atomic.AtomicReference;
 
 import org.apache.pulsar.client.api.PulsarClient;
 import org.apache.pulsar.client.api.Schema;
@@ -31,6 +32,7 @@
 import org.springframework.core.log.LogAccessor;
 import org.springframework.lang.Nullable;
 import org.springframework.pulsar.core.DefaultTopicResolver;
+import org.springframework.pulsar.core.PulsarClientProxy;
 import org.springframework.pulsar.core.TopicResolver;
 import org.springframework.util.Assert;
 import org.springframework.util.CollectionUtils;
@@ -42,10 +44,15 @@
  * @author Christophe Bornet
  * @author Chris Bono
  */
-public final class DefaultReactivePulsarSenderFactory<T> implements ReactivePulsarSenderFactory<T> {
+public final class DefaultReactivePulsarSenderFactory<T>
+		implements ReactivePulsarSenderFactory<T>, RestartableComponentSupport {
+
+	private static final int LIFECYCLE_PHASE = (Integer.MIN_VALUE / 2) - 100;
 
 	private final LogAccessor logger = new LogAccessor(this.getClass());
 
+	private final AtomicReference<State> currentState = RestartableComponentSupport.initialState();
+
 	private final ReactivePulsarClient reactivePulsarClient;
 
 	private final TopicResolver topicResolver;
@@ -110,6 +117,9 @@ public ReactiveMessageSender<T> createSender(Schema<T> schema, @Nullable String
 	private ReactiveMessageSender<T> doCreateReactiveMessageSender(Schema<T> schema, @Nullable String topic,
 			@Nullable List<ReactiveMessageSenderBuilderCustomizer<T>> customizers) {
 		Objects.requireNonNull(schema, "Schema must be specified");
+
+		this.logger.warn(() -> "**** Du CreateMessageSender for topic=" + topic);
+
 		String resolvedTopic = this.topicResolver.resolveTopic(topic, () -> getDefaultTopic()).orElseThrow();
 		this.logger.trace(() -> "Creating reactive message sender for '%s' topic".formatted(resolvedTopic));
 
@@ -139,6 +149,41 @@ public String getDefaultTopic() {
 		return this.defaultTopic;
 	}
 
+	/**
+	 * Return the phase that this lifecycle object is supposed to run in.
+	 * <p>
+	 * This component has a phase that comes after the {@link PulsarClientProxy
+	 * restartable client} but before other lifecycle and smart lifecycle components whose
+	 * phase values are &quot;0&quot; and &quot;max&quot;, respectively.
+	 * @return a phase that is after the restartable client and before other default
+	 * components.
+	 * @see PulsarClientProxy#getPhase()
+	 */
+	@Override
+	public int getPhase() {
+		return LIFECYCLE_PHASE;
+	}
+
+	@Override
+	public AtomicReference<State> currentState() {
+		return this.currentState;
+	}
+
+	@Override
+	public LogAccessor logger() {
+		return this.logger;
+	}
+
+	@Override
+	public void doStop() {
+		try {
+			this.reactiveMessageSenderCache.close();
+		}
+		catch (Exception e) {
+			throw new RuntimeException(e);
+		}
+	}
+
 	/**
 	 * Builder for {@link DefaultReactivePulsarSenderFactory}.
 	 *

spring-pulsar-reactive/src/main/java/org/springframework/pulsar/reactive/core/RestartableComponentSupport.java

@@ -0,0 +1,138 @@
+/*
+ * Copyright 2022-2023 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.pulsar.reactive.core;
+
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.springframework.beans.factory.DisposableBean;
+import org.springframework.context.SmartLifecycle;
+import org.springframework.core.log.LogAccessor;
+import org.springframework.lang.Nullable;
+
+/**
+ * Provides a simple base implementation for a component that can be restarted (stopped
+ * then started) and still be in a usable state.
+ * <p>
+ * This is an interface that provides default methods that rely on the current component
+ * state which must be maintained by the implementing component.
+ * <p>
+ * This can serve as a base implementation for coordinated checkpoint and restore by
+ * simply implementing the {@link #doStart() start} and/or {@link #doStop() stop} callback
+ * to re-acquire and release resources, respectively.
+ * <p>
+ * Implementors are required to provide the component state and a logger.
+ *
+ * @author Chris Bono
+ */
+interface RestartableComponentSupport extends SmartLifecycle, DisposableBean {
+
+	/**
+	 * Gets the initial state for the implementing component.
+	 * @return the initial component state
+	 */
+	static AtomicReference<State> initialState() {
+		return new AtomicReference<>(State.CREATED);
+	}
+
+	/**
+	 * Callback to get the current state from the component.
+	 * @return the current state of the component
+	 */
+	AtomicReference<State> currentState();
+
+	/**
+	 * Callback to get the component specific logger.
+	 * @return the component specific logger
+	 */
+	LogAccessor logger();
+
+	/**
+	 * Lifecycle state of this factory.
+	 */
+	enum State {
+
+		/** Component initially created. */
+		CREATED,
+		/** Component in the process of being started. */
+		STARTING,
+		/** Component has been started. */
+		STARTED,
+		/** Component in the process of being stopped. */
+		STOPPING,
+		/** Component has been stopped. */
+		STOPPED,
+		/** Component has been destroyed. */
+		DESTROYED;
+
+	}
+
+	@Override
+	default boolean isRunning() {
+		return State.STARTED.equals(currentState().get());
+	}
+
+	@Override
+	default void start() {
+		State current = currentState().getAndUpdate(state -> isCreatedOrStopped(state) ? State.STARTING : state);
+		if (isCreatedOrStopped(current)) {
+			logger().debug(() -> "Starting...");
+			doStart();
+			currentState().set(State.STARTED);
+			logger().debug(() -> "Started");
+		}
+	}
+
+	private static boolean isCreatedOrStopped(@Nullable State state) {
+		return State.CREATED.equals(state) || State.STOPPED.equals(state);
+	}
+
+	/**
+	 * Callback invoked during startup - default implementation does nothing.
+	 */
+	default void doStart() {
+	}
+
+	@Override
+	default void stop() {
+		State current = currentState().getAndUpdate(state -> isCreatedOrStarted(state) ? State.STOPPING : state);
+		if (isCreatedOrStarted(current)) {
+			logger().debug(() -> "Stopping...");
+			doStop();
+			currentState().set(State.STOPPED);
+			logger().debug(() -> "Stopped");
+		}
+	}
+
+	private static boolean isCreatedOrStarted(@Nullable State state) {
+		return State.CREATED.equals(state) || State.STARTED.equals(state);
+	}
+
+	/**
+	 * Callback invoked during stop - default implementation does nothing.
+	 */
+	default void doStop() {
+	}
+
+	@Override
+	default void destroy() {
+		logger().debug(() -> "Destroying...");
+		stop();
+		currentState().set(State.DESTROYED);
+		logger().debug(() -> "Destroyed");
+	}
+
+}

spring-pulsar-reactive/src/test/java/org/springframework/pulsar/reactive/core/DefaultReactivePulsarSenderFactoryTests.java

@@ -20,8 +20,12 @@
 import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException;
 import static org.assertj.core.api.Assertions.assertThatNullPointerException;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.clearInvocations;
 import static org.mockito.Mockito.inOrder;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
 
 import java.util.Arrays;
 import java.util.Collections;
@@ -210,4 +214,25 @@ private ReactivePulsarSenderFactory<String> newSenderFactoryWithCustomizers(
 
 	}
 
+	@Nested
+	class RestartFactoryTests {
+
+		@Test
+		void restartLifecycle() throws Exception {
+			var cache = spy(AdaptedReactivePulsarClientFactory.createCache());
+			var senderFactory = (DefaultReactivePulsarSenderFactory<String>) newSenderFactoryWithCache(cache);
+			senderFactory.start();
+			senderFactory.createSender(schema, "topic1");
+			senderFactory.stop();
+			senderFactory.stop();
+			verify(cache, times(1)).close();
+			clearInvocations(cache);
+			senderFactory.start();
+			senderFactory.createSender(schema, "topic2");
+			senderFactory.stop();
+			verify(cache, times(1)).close();
+		}
+
+	}
+
 }

spring-pulsar/src/main/java/org/springframework/pulsar/core/CachingPulsarProducerFactory.java

@@ -23,6 +23,7 @@
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 
 import org.apache.pulsar.client.api.MessageId;
@@ -58,12 +59,17 @@
  * @author Alexander Preuß
  * @author Christophe Bornet
  */
-public class CachingPulsarProducerFactory<T> extends DefaultPulsarProducerFactory<T> implements DisposableBean {
+public class CachingPulsarProducerFactory<T> extends DefaultPulsarProducerFactory<T>
+		implements RestartableComponentSupport {
+
+	private static final int LIFECYCLE_PHASE = (Integer.MIN_VALUE / 2) - 100;
 
 	private final LogAccessor logger = new LogAccessor(this.getClass());
 
 	private final CacheProvider<ProducerCacheKey<T>, Producer<T>> producerCache;
 
+	private final AtomicReference<State> currentState = RestartableComponentSupport.initialState();
+
 	/**
 	 * Construct a caching producer factory with the specified values for the cache
 	 * configuration.
@@ -85,7 +91,7 @@ public CachingPulsarProducerFactory(PulsarClient pulsarClient, @Nullable String
 				(key, producer, cause) -> {
 					this.logger.debug(() -> "Producer %s evicted from cache due to %s"
 						.formatted(ProducerUtils.formatProducer(producer), cause));
-					closeProducer(producer);
+					closeProducer(producer, true);
 				});
 	}
 
@@ -113,22 +119,51 @@ private Producer<T> createCacheableProducer(Schema<T> schema, String topic,
 		}
 	}
 
+	/**
+	 * Return the phase that this lifecycle object is supposed to run in.
+	 * <p>
+	 * Because this object depends on the restartable client, it uses a phase slightly
+	 * larger than the one used by the restartable client. This ensures that it starts
+	 * after and stops before the restartable client.
+	 * @return the phase to execute in (just after the restartable client)
+	 * @see PulsarClientProxy#getPhase()
+	 */
+	@Override
+	public int getPhase() {
+		return LIFECYCLE_PHASE;
+	}
+
 	@Override
-	public void destroy() {
-		this.producerCache.invalidateAll((key, producer) -> closeProducer(producer));
+	public AtomicReference<State> currentState() {
+		return this.currentState;
 	}
 
-	private void closeProducer(Producer<T> producer) {
+	@Override
+	public LogAccessor logger() {
+		return this.logger;
+	}
+
+	@Override
+	public void doStop() {
+		this.producerCache.invalidateAll((key, producer) -> closeProducer(producer, false));
+	}
+
+	private void closeProducer(Producer<T> producer, boolean async) {
 		Producer<T> actualProducer = null;
 		if (producer instanceof ProducerWithCloseCallback<T> wrappedProducer) {
 			actualProducer = wrappedProducer.getActualProducer();
 		}
 		if (actualProducer == null) {
-			this.logger.warn(() -> "Unable to get actual producer for %s - will skip closing it"
+			this.logger.trace(() -> "Unable to get actual producer for %s - will skip closing it"
 				.formatted(ProducerUtils.formatProducer(producer)));
 			return;
 		}
-		ProducerUtils.closeProducerAsync(actualProducer, this.logger);
+		if (async) {
+			ProducerUtils.closeProducerAsync(actualProducer, this.logger);
+		}
+		else {
+			ProducerUtils.closeProducer(actualProducer, this.logger, Duration.ofSeconds(15L));
+		}
 	}
 
 	/**