- html - 出于某种原因,IE8 对我的 Sass 文件中继承的 html5 CSS 不友好?
- JMeter 在响应断言中使用 span 标签的问题
- html - 在 :hover and :active? 上具有不同效果的 CSS 动画
- html - 相对于居中的 html 内容固定的 CSS 重复背景?
所以我试图让交互式查询与 Kafka 流一起工作。我有 Zookeeper 和 Kafka 在本地运行(在 Windows 上)。我使用 C:\temp 作为存储文件夹,用于 Zookeeper 和 Kafka。
我已经设置了这样的主题
kafka-topics.bat --zookeeper localhost:2181 --create --replication-factor 1 --partitions 1 --topic rating-submit-topic
kafka-topics.bat --zookeeper localhost:2181 --create --replication-factor 1 --partitions 1 --topic rating-output-topic
KTable
.实际的流处理似乎按预期进行,我可以打印
KTable
的结果并且它们与针对该主题产生的内容相匹配。
Streams.allMetadata()
时似乎会出现此问题。方法,它返回一个空列表。
package Processing.Ratings {
import java.util.concurrent.TimeUnit
import Entities.Ranking
import Serialization.JSONSerde
import Topics.RatingsTopics
import scala.util.Random
import org.apache.kafka.clients.producer.ProducerRecord
import org.apache.kafka.clients.producer.KafkaProducer
import org.apache.kafka.common.serialization.Serdes
import Utils.Settings
import org.apache.kafka.clients.producer.ProducerConfig
object RatingsProducerApp extends App {
run()
private def run(): Unit = {
val jSONSerde = new JSONSerde[Ranking]
val random = new Random
val producerProps = Settings.createBasicProducerProperties
val rankingList = List(
Ranking("jarden@here.com","sacha@here.com", 1.5f),
Ranking("miro@here.com","mary@here.com", 1.5f),
Ranking("anne@here.com","margeret@here.com", 3.5f),
Ranking("frank@here.com","bert@here.com", 2.5f),
Ranking("morgan@here.com","ruth@here.com", 1.5f))
producerProps.put(ProducerConfig.ACKS_CONFIG, "all")
System.out.println("Connecting to Kafka cluster via bootstrap servers " +
s"${producerProps.getProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG)}")
// send a random string from List event every 100 milliseconds
val rankingProducer = new KafkaProducer[String, Array[Byte]](
producerProps, Serdes.String.serializer, Serdes.ByteArray.serializer)
//while (true) {
for (i <- 0 to 10) {
val ranking = rankingList(random.nextInt(rankingList.size))
val rankingBytes = jSONSerde.serializer().serialize("", ranking)
System.out.println(s"Writing ranking ${ranking} to input topic ${RatingsTopics.RATING_SUBMIT_TOPIC}")
rankingProducer.send(new ProducerRecord[String, Array[Byte]](
RatingsTopics.RATING_SUBMIT_TOPIC, ranking.toEmail, rankingBytes))
Thread.sleep(100)
}
Runtime.getRuntime.addShutdownHook(new Thread(() => {
rankingProducer.close(10, TimeUnit.SECONDS)
}))
}
}
}
def createRatingStreamsProperties() : Properties = {
val props = createBasicStreamProperties
props.put(StreamsConfig.APPLICATION_ID_CONFIG, "ratings-application")
props.put(StreamsConfig.CLIENT_ID_CONFIG, "ratings-application-client")
props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass)
props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass)
props
}
private def createBasicStreamProperties() : Properties = {
val props = new Properties()
props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, bootStrapServers)
props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass)
props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass)
// Records should be flushed every 10 seconds. This is less than the default
// in order to keep this example interactive.
props.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 10000.asInstanceOf[Object])
// For illustrative purposes we disable record caches
props.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0.asInstanceOf[Object])
props
}
import java.util.Properties
import java.util.concurrent.TimeUnit
import org.apache.kafka.common.serialization._
import org.apache.kafka.streams._
import org.apache.kafka.streams.kstream._
import Entities.Ranking
import Serialization.JSONSerde
import Topics.RatingsTopics
import Utils.Settings
package Processing.Ratings {
import Stores.StateStores
import org.apache.kafka.streams.state.HostInfo
class DummyRankingReducer extends Reducer[Ranking] {
override def apply(value1: Ranking, value2: Ranking): Ranking = {
value2
}
}
class RankingByEmailInitializer extends Initializer[List[Ranking]] {
override def apply(): List[Ranking] = List[Ranking]()
}
class RankingByEmailAggregator extends Aggregator[String, Ranking,List[Ranking]] {
override def apply(aggKey: String, value: Ranking, aggregate: List[Ranking]) = {
value :: aggregate
}
}
object RatingStreamProcessingApp extends App {
run()
private def run() : Unit = {
val stringSerde = Serdes.String
val rankingSerde = new JSONSerde[Ranking]
val listRankingSerde = new JSONSerde[List[Ranking]]
val builder: KStreamBuilder = new KStreamBuilder
val rankings = builder.stream(stringSerde, rankingSerde, RatingsTopics.RATING_SUBMIT_TOPIC)
val rankingTable = rankings.groupByKey(stringSerde,rankingSerde)
.aggregate(
new RankingByEmailInitializer(),
new RankingByEmailAggregator(),
listRankingSerde,
StateStores.RANKINGS_BY_EMAIL_STORE
)
rankingTable.toStream.print()
val streams: KafkaStreams = new KafkaStreams(builder, Settings.createRatingStreamsProperties)
val restEndpoint:HostInfo = new HostInfo(Settings.restApiDefaultHostName, Settings.restApiDefaultPort)
System.out.println(s"Connecting to Kafka cluster via bootstrap servers ${Settings.bootStrapServers}")
System.out.println(s"REST endpoint at http://${restEndpoint.host}:${restEndpoint.port}")
// Always (and unconditionally) clean local state prior to starting the processing topology.
// We opt for this unconditional call here because this will make it easier for you to play around with the example
// when resetting the application for doing a re-run (via the Application Reset Tool,
// http://docs.confluent.io/current/streams/developer-guide.html#application-reset-tool).
//
// The drawback of cleaning up local state prior is that your app must rebuilt its local state from scratch, which
// will take time and will require reading all the state-relevant data from the Kafka cluster over the network.
// Thus in a production scenario you typically do not want to clean up always as we do here but rather only when it
// is truly needed, i.e., only under certain conditions (e.g., the presence of a command line flag for your app).
// See `ApplicationResetExample.java` for a production-like example.
//streams.cleanUp();
streams.start()
val restService = new RatingRestService(streams, restEndpoint)
restService.start()
//****************************************************************
// WHY DOES METADATA NOT WORK WHEN THERE IS CLEARLY A STORE IN USE
// WHY DOES METADATA NOT WORK WHEN THERE IS CLEARLY A STORE IN USE
// WHY DOES METADATA NOT WORK WHEN THERE IS CLEARLY A STORE IN USE
// WHY DOES METADATA NOT WORK WHEN THERE IS CLEARLY A STORE IN USE
//****************************************************************
val SIZE = streams.allMetadata.size()
val SIZE2 = streams.allMetadataForStore(StateStores.RANKINGS_BY_EMAIL_STORE).size()
import org.apache.kafka.streams.state.KeyValueIterator
import org.apache.kafka.streams.state.QueryableStoreTypes
import org.apache.kafka.streams.state.ReadOnlyKeyValueStore
val keyValueStore = streams.store(StateStores.RANKINGS_BY_EMAIL_STORE, QueryableStoreTypes.keyValueStore)
val range = keyValueStore.all
val HASNEXT = range.hasNext
import org.apache.kafka.streams.KeyValue
while (range.hasNext ) {
val next = range.next
System.out.println(String.format("key: %s | value: %s", next.key, next.value))
}
Runtime.getRuntime.addShutdownHook(new Thread(() => {
streams.close(10, TimeUnit.SECONDS)
restService.stop
}))
//return unit
()
}
}
kafka {
bootStrapServers = "localhost:9092"
zooKeepers = "zookeeper:2181"
schemaRegistryUrl = "http://localhost:8081"
partition = 0,
restApiDefaultHostName = "localhost",
restApiDefaultPort = "8080"
}
package Processing.Ratings
import org.apache.kafka.streams.KafkaStreams
import org.apache.kafka.streams.state.StreamsMetadata
import java.util.stream.Collectors
import Entities.HostStoreInfo
import org.apache.kafka.common.serialization.Serializer
import org.apache.kafka.connect.errors.NotFoundException
import scala.collection.JavaConverters._
/**
* Looks up StreamsMetadata from KafkaStreams
*/
class MetadataService(val streams: KafkaStreams) {
/**
* Get the metadata for all of the instances of this Kafka Streams application
*
* @return List of { @link HostStoreInfo}
*/
def streamsMetadata() : List[HostStoreInfo] = {
// Get metadata for all of the instances of this Kafka Streams application
val metadata = streams.allMetadata
return mapInstancesToHostStoreInfo(metadata)
}
/**
* Get the metadata for all instances of this Kafka Streams application that currently
* has the provided store.
*
* @param store The store to locate
* @return List of { @link HostStoreInfo}
*/
def streamsMetadataForStore(store: String) : List[HostStoreInfo] = {
// Get metadata for all of the instances of this Kafka Streams application hosting the store
val metadata = streams.allMetadataForStore(store)
return mapInstancesToHostStoreInfo(metadata)
}
/**
* Find the metadata for the instance of this Kafka Streams Application that has the given
* store and would have the given key if it exists.
*
* @param store Store to find
* @param key The key to find
* @return { @link HostStoreInfo}
*/
def streamsMetadataForStoreAndKey[T](store: String, key: T, serializer: Serializer[T]) : HostStoreInfo = {
// Get metadata for the instances of this Kafka Streams application hosting the store and
// potentially the value for key
val metadata = streams.metadataForKey(store, key, serializer)
if (metadata == null)
throw new NotFoundException(
s"No metadata could be found for store : ${store}, and key type : ${key.getClass.getName}")
return new HostStoreInfo(metadata.host, metadata.port, metadata.stateStoreNames.asScala.toList)
}
def mapInstancesToHostStoreInfo(metadatas : java.util.Collection[StreamsMetadata]) : List[HostStoreInfo] = {
metadatas.stream.map[HostStoreInfo](metadata =>
HostStoreInfo(
metadata.host(),
metadata.port,
metadata.stateStoreNames.asScala.toList))
.collect(Collectors.toList())
.asScala.toList
}
}
package Processing.Ratings
import org.apache.kafka.streams.KafkaStreams
import org.apache.kafka.streams.state.HostInfo
import akka.actor.ActorSystem
import akka.http.scaladsl.Http
import akka.http.scaladsl.model._
import akka.http.scaladsl.server.Directives._
import akka.stream.ActorMaterializer
import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._
import spray.json.DefaultJsonProtocol._
import Entities.AkkaHttpEntitiesJsonFormats._
import Entities._
import akka.http.scaladsl.marshalling.ToResponseMarshallable
import scala.concurrent.Future
object RestService {
val DEFAULT_REST_ENDPOINT_HOSTNAME = "localhost"
}
class RatingRestService(val streams: KafkaStreams, val hostInfo: HostInfo) {
val metadataService = new MetadataService(streams)
var bindingFuture: Future[Http.ServerBinding] = null
implicit val system = ActorSystem("rating-system")
implicit val materializer = ActorMaterializer()
implicit val executionContext = system.dispatcher
def start() : Unit = {
val emailRegexPattern = """\w+""".r
val route =
path("ratingByEmail" / emailRegexPattern) { email =>
get {
//TODO : This would come from Kafka store, either local or remote
complete(ToResponseMarshallable.apply(List[Ranking](
Ranking("fred@here.com", "sacha@there.com", 4.0f),
Ranking("sam@here.com", "sacha@there.com", 2.0f)))
)
}
} ~
path("instances") {
get {
val x = metadataService.streamsMetadata
complete(ToResponseMarshallable.apply(metadataService.streamsMetadata))
}
}
bindingFuture = Http().bindAndHandle(route, hostInfo.host, hostInfo.port)
println(s"Server online at http://${hostInfo.host}:${hostInfo.port}/\n")
Runtime.getRuntime.addShutdownHook(new Thread(() => {
bindingFuture
.flatMap(_.unbind()) // trigger unbinding from the port
.onComplete(_ => system.terminate()) // and shutdown when done
}))
}
def stop() : Unit = {
bindingFuture
.flatMap(_.unbind()) // trigger unbinding from the port
.onComplete(_ => system.terminate()) // and shutdown when done
}
def thisHost(hostStoreInfo: HostStoreInfo) : Boolean = {
hostStoreInfo.host.equals(hostInfo.host()) &&
hostStoreInfo.port == hostInfo.port
}
}
KTable
看看结果如何正在显示,然后我启动了生产者并通过流获取的主题推送了更多消息
localhost:8080/instances
获取元数据时, 我得到的只是一个空列表
[]
val SIZE = streams.allMetadata.size()
val SIZE2 = streams.allMetadataForStore(StateStores.RANKINGS_BY_EMAIL_STORE).size()
import org.apache.kafka.streams.state.KeyValueIterator
import org.apache.kafka.streams.state.QueryableStoreTypes
import org.apache.kafka.streams.state.ReadOnlyKeyValueStore
val keyValueStore = streams.store(StateStores.RANKINGS_BY_EMAIL_STORE, QueryableStoreTypes.keyValueStore)
val range = keyValueStore.all
val HASNEXT = range.hasNext
import org.apache.kafka.streams.KeyValue
while (range.hasNext ) {
val next = range.next
System.out.println(String.format("key: %s | value: %s", next.key, next.value))
}
最佳答案
所以我想通了,原来是由于缺少这个配置值
props.put(StreamsConfig.APPLICATION_SERVER_CONFIG, "localhost:8080")
http://localhost:8080/instance
开始工作。但后来我开始收到这个奇怪的异常
org.apache.kafka.streams.errors.InvalidStateStoreException: the state store, my-key-value-store, may have migrated to another instance.
at org.apache.kafka.streams.state.internals.StreamThreadStateStoreProvider.stores(StreamThreadStateStoreProvider.java:49)
at org.apache.kafka.streams.state.internals.QueryableStoreProvider.getStore(QueryableStoreProvider.java:55)
at org.apache.kafka.streams.KafkaStreams.store(KafkaStreams.java:699)
package Utils
import scala.concurrent._
import scala.concurrent.duration._
object Retry {
/**
* exponential back off for retry
*/
def exponentialBackoff(r: Int): Duration = scala.math.pow(2, r).round * 500 milliseconds
def noIgnore(t: Throwable): Boolean = false
/**
* retry a particular block that can fail
*
* @param maxRetry how many times to retry before to giveup
* @param deadline how long to retry before giving up; default None
* @param backoff a back-off function that returns a Duration after which to retry. default is an exponential backoff at 100 milliseconds steps
* @param ignoreThrowable if you want to stop retrying on a particular exception
* @param block a block of code to retry
* @param ctx an execution context where to execute the block
* @returns an eventual Future succeeded with the value computed or failed with one of:
* `TooManyRetriesException` if there were too many retries without an exception being caught. Probably impossible if you pass decent parameters
* `DeadlineExceededException` if the retry didn't succeed before the provided deadline
* `TimeoutException` if you provide a deadline and the block takes too long to execute
* `Throwable` the last encountered exception
*/
def retry[T](maxRetry: Int,
deadline: Option[Deadline] = None,
backoff: (Int) => Duration = exponentialBackoff,
ignoreThrowable: Throwable => Boolean = noIgnore)(block: => T)(implicit ctx: ExecutionContext): Future[T] = {
class TooManyRetriesException extends Exception("too many retries without exception")
class DeadlineExceededException extends Exception("deadline exceded")
val p = Promise[T]
def recursiveRetry(retryCnt: Int, exception: Option[Throwable])(f: () => T): Option[T] = {
if (maxRetry == retryCnt
|| deadline.isDefined && deadline.get.isOverdue) {
exception match {
case Some(t) =>
p failure t
case None if deadline.isDefined && deadline.get.isOverdue =>
p failure (new DeadlineExceededException)
case None =>
p failure (new TooManyRetriesException)
}
None
} else {
val success = try {
val rez = if (deadline.isDefined) {
Await.result(future(f()), deadline.get.timeLeft)
} else {
f()
}
Some(rez)
} catch {
case t: Throwable if !ignoreThrowable(t) =>
blocking {
val interval = backoff(retryCnt).toMillis
Thread.sleep(interval)
}
recursiveRetry(retryCnt + 1, Some(t))(f)
case t: Throwable =>
p failure t
None
}
success match {
case Some(v) =>
p success v
Some(v)
case None => None
}
}
}
def doBlock() = block
Future {
recursiveRetry(0, None)(doBlock)
}
p.future
}
}
def printStoreMetaData(streams:KafkaStreams) : Unit = {
import org.apache.kafka.streams.state.KeyValueIterator
import org.apache.kafka.streams.state.QueryableStoreTypes
import org.apache.kafka.streams.state.ReadOnlyKeyValueStore
val keyValueStoreTry = waitUntilStoreIsQueryable(
StateStores.RANKINGS_BY_EMAIL_STORE,
QueryableStoreTypes.keyValueStore[String,List[Ranking]](),
streams
) match {
case Success(keyValueStore) => {
val SIZE = streams.allMetadata.size()
val SIZE2 = streams.allMetadataForStore(StateStores.RANKINGS_BY_EMAIL_STORE).size()
val range = keyValueStore.all
val HASNEXT = range.hasNext
import org.apache.kafka.streams.KeyValue
while (range.hasNext ) {
val next = range.next
System.out.println(String.format("key: %s | value: %s", next.key, next.value))
}
}
case Failure(f) => println(f)
}
}
关于apache-kafka - Kafkastreams.allMetadata() 方法返回空列表,我们在Stack Overflow上找到一个类似的问题: https://stackoverflow.com/questions/46054763/
在流处理方面,Apache Beam和Apache Kafka之间有什么区别? 我也试图掌握技术和程序上的差异。 请通过您的经验报告来帮助我理解。 最佳答案 Beam是一种API,它以一种统一的方式使
有点n00b的问题。 如果我使用 Apache Ignite 进行消息传递和事件处理,是否还需要使用 Kafka? 与 Ignite 相比,Kafka 基本上会给我哪些(如果有的话)额外功能? 提前致
Apache MetaModel 是一个数据访问框架,它为发现、探索和查询不同类型的数据源提供了一个通用接口(interface)。 Apache Drill 是一种无架构的 SQL 查询引擎,它通过
Tomcat是一个广泛使用的java web服务器,而Apache也是一个web服务器,它们在实际项目使用中有什么不同? 经过一些研究,我有了一个简单的想法,比如, Apache Tomcat Ja
既然简单地使用 Apache 就足以运行许多 Web 应用程序,那么人们何时以及为什么除了 Apache 之外还使用 Tomcat? 最佳答案 Apache Tomcat是一个网络服务器和 Java
我在某个 VPS( friend 的带 cPanel 的 apache 服务器)上有一个帐户,我在那里有一个 public_html 目录。我们有大约 5-6 个网站: /home/myusernam
我目前正在尝试将模块加载到 Apache,使用 cmake 构建。该模块称为 mod_mapcache。它已成功构建并正确安装在/usr/lib/apache2/modules directroy 中
我对 url 中的问号有疑问。 例如:我有 url test.com/controller/action/part_1%3Fpart_2 (其中 %3F 是 url 编码的问号),并使用此重写规则:R
在同一台机器上,Apache 在端口 80 上运行,Tomcat 在端口 8080 上运行。 Apache 包括 html;css;js;文件并调用 tomcat 服务。 基本上 exampledom
Apache 1 和 Apache 2 的分支有什么区别? 使用一种或另一种的优点和缺点? 似乎 Apache 2 的缺点之一是使用大量内存,但也许它处理请求的速度更快? 最有趣的是 Apache 作
实际上,我们正在使用 Apache 网络服务器来托管我们的 REST-API。 脚本是用 Lua 编写的,并使用 mod-lua 映射。 例如来自 httpd.conf 的实际片段: [...] Lu
我在 apache 上的 ubuntu 中有一个虚拟主机,这不是我的主要配置,我有另一个网页作为我的主要网页,所以我想使用虚拟主机在同一个 IP 上设置这个。 urologyexpert.mx 是我的
我使用 Apache camel 已经很长时间了,发现它是满足各种系统集成相关业务需求的绝佳解决方案。但是几年前我遇到了 Apache Nifi 解决方案。经过一番谷歌搜索后,我发现虽然 Nifi 可
由于两者都是一次处理事件的流框架,这两种技术/流框架之间的核心架构差异是什么? 此外,在哪些特定用例中,一个比另一个更合适? 最佳答案 正如您所提到的,两者都是实时内存计算的流式平台。但是当您仔细观察
apache 文件(如 httpd.conf 和虚拟主机)中使用的语言名称是什么,例如 # Ensure that Apache listens on port 80 Listen 80 D
作为我学习过程的一部分,我认为如果我扩展更多关于 apache 的知识会很好。我有几个问题,虽然我知道有些内容可能需要相当冗长的解释,但我希望您能提供一个概述,以便我知道去哪里寻找。 (最好引用 mo
关闭。这个问题是opinion-based .它目前不接受答案。 想改善这个问题吗?更新问题,以便可以通过 editing this post 用事实和引文回答问题. 4 个月前关闭。 Improve
就目前而言,这个问题不适合我们的问答形式。我们希望答案得到事实、引用或专业知识的支持,但这个问题可能会引起辩论、争论、投票或扩展讨论。如果您觉得这个问题可以改进并可能重新打开,visit the he
这个问题在这里已经有了答案: Difference Between Apache Kafka and Camel (Broker vs Integration) (4 个回答) 3年前关闭。 据我所知
我有 2 个使用相同规则的子域,如下所示: RewriteEngine On RewriteCond %{REQUEST_FILENAME} !-f RewriteCond
我是一名优秀的程序员,十分优秀!