Screenshot CC BY-NC 3.0 Before: the mess CC BY-NC 3.0 Before: - PowerPoint PPT Presentation

🚬 🚬

Screenshot CC BY-NC 3.0

Before: the mess CC BY-NC 3.0

Before: the mess CC BY-NC 3.0

Before: the mess Akka & Spray RabbitMQ C++ & CUDA Cassandra CC BY-NC 3.0

Before: the mess scene topic (Map[String, String], Array[Byte]) (Map[String, String], Array[Byte]) identity topic CC BY-NC 3.0

Before: the mess scene topic _tmp_ x topics (Map[String, String], Array[Byte]) JSON (Map[String, String], Array[Byte]) identity topic CC BY-NC 3.0

Before: the mess fire-and-forget at-most-once at-most-once fire-and-forget non-durable CC BY-NC 3.0

Asynchronous request–response orchestration class Orchestrator extends Actor with ActorFSM[Orchestrator.State, Orchestrator.Data] { startWith(Idle, UninitializedData) when(Idle, idleTimeout)(idleSF) when(ImageProcessing, stepTimeout)(imageProcessingSF) when(WaitingForProcessingResult, stepTimeout)(waitingForProcessingSF) whenUnhandled(timeoutSF) onTransition { case _ -> Aborted => ??? ... } def idleSF: StateFunction = ??? def imageProcessingSF: StateFunction = ??? def waitingForProcessingSF: StateFunction = ??? def timeoutSF: StateFunction = { case Event(StateTimeout, data: RunningTransactionData) => goto(Aborted) } } CC BY-NC 3.0

After: proper microservices text scene dashboard ingest identity CC BY-NC 3.0

After: proper microservices text scene dashboard ingest identity CC BY-NC 3.0

After: proper microservices Lagom Cassandra Akka text scene dashboard ingest identity Kafka CC BY-NC 3.0

After: proper microservices text group text topic scene topic text scene dashboard ingest identity tweet-image topic scene group identity topic identity group CC BY-NC 3.0

After: proper microservices message Text { … } text topic message Envelope { scene topic int32 version = 1; string handle = 2; message Scene { … } int64 processingTimestamp = 3; int64 ingestionTimestamp = 4; text string correlationId = 5; string messageId = 6; string messageType = 7; bytes payload = 8; } scene dashboard ingest identity tweet-image topic bytes image; identity topic message Identity { … } CC BY-NC 3.0

After: proper microservices at-least-once fire-and-forget at-most-once text scene dashboard ingest identity at-least-once at-least-once CC BY-NC 3.0

Persistence and formats message Envelope { 
 int32 version = 1; 
 string handle = 2; 
 int64 processingTimestamp = 3; 
 int64 ingestionTimestamp = 4; 
 string correlationId = 5; 
 string messageId = 6; 
 string messageType = 7; 
 bytes payload = 8; 
 } CC BY-NC 3.0

Persistence and formats message Text { repeated string areas = 1; 
 } CC BY-NC 3.0


 Persistence and formats message Scene { message Label { 
 string label = 1; 
 double score = 2; 
 } repeated Label labels = 3; 
 } CC BY-NC 3.0


 
 Persistence and formats message Identity { 
 oneof face { 
 IdentifiedFace identifiedFace = 1; 
 UnknownFace unknownFace = 2; 
 } 
 message IdentifiedFace { 
 string name = 1; 
 double score = 2; 
 } 
 message UnknownFace { } 
 } CC BY-NC 3.0


 
 Persistence and formats message IdentifyFace { 
 int64 ingestionTimestamp = 2; 
 string correlationId = 3; 
 string handle = 4; 
 bytes image = 5; 
 } 
 message IdentifyFaces { 
 repeated IdentifyFace identifyFaces = 1; 
 } 
 message FaceImage { 
 double confidence = 1; 
 int32 x = 2; 
 int32 y = 3; 
 int32 w = 4; 
 int32 h = 5; 
 bytes rgbBitmap = 6; 
 } CC BY-NC 3.0

Fire–and–forget send object Act { def props(config: Config): Props = { val producerConf = KafkaProducer.Conf (config.getConfig("..."), new StringSerializer, KafkaSerializer[Envelope](_.toByteArray)) Props( classOf [Act], producerConf) } } class Act(producerConf: KafkaProducer.Conf[String, Envelope]) extends Actor { private [ this ] val producer = KafkaProducer(conf = producerConf) override def receive: Receive = { case TweetImage (handle, content) => producer.send(KafkaProducerRecord("tweet-image", handle, Envelope (version = 100, handle = handle, ingestionTimestamp = System. nanoTime() , processingTimestamp = System. nanoTime() , messageId = UUID. randomUUID() .toString, correlationId = UUID. randomUUID() .toString, payload = content))) } } CC BY-NC 3.0

At least once delivery I class SceneClassifierActor(…) extends Actor { private[this] val kafkaConsumerActor = context.actorOf(…) private[this] val producer = KafkaProducer(…) override def receive: Receive = { case extractor(consumerRecords) => val futures = consumerRecords.pairs.flatMap { case (_, envelope) => val outEnvelope = … Some(producer.send(KafkaProducerRecord("scene", envelope.handle, outEnvelope))) } import context.dispatcher Future. sequence (futures).onSuccess { case _ => kafkaConsumerActor ! Confirm(consumerRecords.offsets, commit = true ) } } } CC BY-NC 3.0

At least once delivery II class IdentityMatcherActor(...) extends PersistentActor with AtLeastOnceDelivery { override val persistenceId: String = "identity-matcher-actor" def identifyFacesAndSend(identifyFaces: Seq[IdentifyFace])( implicit executor: ExecutionContext): Future[Unit] = { // Future.sequence(producer.send(...)) } def handleIdentifyFace: Receive = { case (deliveryId: Long, identifyFaces: IdentifyFaces) => import context.dispatcher identifyFacesAndSend(identifyFaces.identifyFaces).onSuccess { case _ => confirmDelivery(deliveryId) } case IdentifyFaces (faces) => import context.dispatcher identifyFacesAndSend(faces).onFailure { case _ => self ! Kill } } override def receiveRecover: Receive = handleIdentifyFace override def receiveCommand: Receive = handleIdentifyFace orElse { case extractor(consumerRecords) => val identifyFaces = consumerRecords.pairs.map { case ( _ , envelope) => IdentifyFace(envelope.ingestionTimestamp, envelope.correlationId, envelope.handle, envelope.payload)) } persist(IdentifyFaces(identifyFaces = identifyFaces)) { result => deliver(self.path)(deliveryId => (deliveryId, result)) sender() ! Confirm(consumerRecords.offsets, commit = true ) } } } CC BY-NC 3.0


 
 
 
 At least once delivery III public interface TweetImageService extends Service { 
 Topic<Envelope> tweetImageTopic(); 
 … 
 } 
 public class TextServiceImpl implements TextService { 
 private final PersistentEntityRegistry persistentEntityRegistry; 
 public Topic<Envelope> textTopic() { 
 return TopicProducer.singleStreamWithOffset(offset -> persistentEntityRegistry 
 .eventStream(TextEntityEvent.OcredTag. INSTANC E, offset) 
 .map(p -> new Pair<>(p.first().envelope(), offset)) 
 ); 
 } 
 @Inject 
 public TextServiceImpl(PersistentEntityRegistry persistentEntityRegistry, TweetImageService tweetImageService) { 
 this .persistentEntityRegistry = persistentEntityRegistry; 
 persistentEntityRegistry.register(TextEntity. class ); 
 tweetImageService.tweetImageTopic().subscribe().withGroupId("text").atLeastOnce(Flow.fromFunction( this ::extractText)); 
 } 
 private Done extractText(Envelope envelope) { 
 TextEntityCommand.Ocr command = new TextEntityCommand.Ocr(…); 
 PersistentEntityRef<TextEntityCommand> ref = persistentEntityRegistry.refFor(TextEntity. class , envelope.handle()); 
 ref.ask(command); 
 return Done.getInstance(); 
 } 
 } CC BY-NC 3.0

At most once delivery class DashboardSinkActor(...) extends Actor { private[this] val kafkaConsumerActor = context.actorOf(...) override def receive: Receive = { case extractor(consumerRecords) => consumerRecords.pairs.foreach { case (_, envelope) => context.system.eventStream.publish(envelope); } kafkaConsumerActor ! Confirm(consumerRecords.offsets, commit = true ) } } CC BY-NC 3.0

ENOUGH SLIDES! CC BY-NC 3.0

After: proper microservices text scene dashboard ingest identity CC BY-NC 3.0

@cakesolutions www.cakesolutions.net enquiries@cakesolutions.net MANCHESTER LONDON NEW YORK CC BY-NC 3.0